diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/simultaneous_translation/utils/monotonic_attention.py b/audio/safeear/safeear_repo/fairseq_ours/examples/simultaneous_translation/utils/monotonic_attention.py deleted file mode 100644 index b5962a632fde6565e45edb9200ade8ebfa95e13d..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/simultaneous_translation/utils/monotonic_attention.py +++ /dev/null @@ -1,180 +0,0 @@ -from typing import Optional - -import torch -from examples.simultaneous_translation.utils.functions import ( - exclusive_cumprod, - moving_sum, - prob_check, -) -from torch import Tensor - - -def expected_alignment_from_p_choose( - p_choose: Tensor, - padding_mask: Optional[Tensor] = None, - eps: float = 1e-6 -): - """ - Calculating expected alignment for from stepwise probability - - Reference: - Online and Linear-Time Attention by Enforcing Monotonic Alignments - https://arxiv.org/pdf/1704.00784.pdf - - q_ij = (1 − p_{ij−1})q_{ij−1} + a+{i−1j} - a_ij = p_ij q_ij - - Parallel solution: - ai = p_i * cumprod(1 − pi) * cumsum(a_i / cumprod(1 − pi)) - - ============================================================ - Expected input size - p_choose: bsz, tgt_len, src_len - """ - prob_check(p_choose) - - # p_choose: bsz, tgt_len, src_len - bsz, tgt_len, src_len = p_choose.size() - dtype = p_choose.dtype - - p_choose = p_choose.float() - - if padding_mask is not None: - p_choose = p_choose.masked_fill(padding_mask.unsqueeze(1), 0.0) - - if p_choose.is_cuda: - p_choose = p_choose.contiguous() - from alignment_train_cuda_binding import alignment_train_cuda as alignment_train - else: - from alignment_train_cpu_binding import alignment_train_cpu as alignment_train - - alpha = p_choose.new_zeros([bsz, tgt_len, src_len]) - alignment_train(p_choose, alpha, eps) - - # Mix precision to prevent overflow for fp16 - alpha = alpha.type(dtype) - - prob_check(alpha) - - return alpha - - -def expected_soft_attention( - alpha: Tensor, - soft_energy: Tensor, - padding_mask: Optional[Tensor] = None, - chunk_size: Optional[int] = None, - eps: float = 1e-10 -): - """ - Function to compute expected soft attention for - monotonic infinite lookback attention from - expected alignment and soft energy. - - Reference: - Monotonic Chunkwise Attention - https://arxiv.org/abs/1712.05382 - - Monotonic Infinite Lookback Attention for Simultaneous Machine Translation - https://arxiv.org/abs/1906.05218 - - alpha: bsz, tgt_len, src_len - soft_energy: bsz, tgt_len, src_len - padding_mask: bsz, src_len - left_padding: bool - """ - if padding_mask is not None: - alpha = alpha.masked_fill(padding_mask.unsqueeze(1), 0.0) - soft_energy = soft_energy.masked_fill( - padding_mask.unsqueeze(1), -float("inf") - ) - - prob_check(alpha) - - dtype = alpha.dtype - - alpha = alpha.float() - soft_energy = soft_energy.float() - - soft_energy = soft_energy - soft_energy.max(dim=2, keepdim=True)[0] - exp_soft_energy = torch.exp(soft_energy) + eps - - if chunk_size is not None: - # Chunkwise - beta = ( - exp_soft_energy - * moving_sum( - alpha / (eps + moving_sum(exp_soft_energy, chunk_size, 1)), - 1, chunk_size - ) - ) - else: - # Infinite lookback - # Notice that infinite lookback is a special case of chunkwise - # where chunksize = inf - inner_items = alpha / (eps + torch.cumsum(exp_soft_energy, dim=2)) - - beta = ( - exp_soft_energy - * torch.cumsum(inner_items.flip(dims=[2]), dim=2) - .flip(dims=[2]) - ) - - if padding_mask is not None: - beta = beta.masked_fill( - padding_mask.unsqueeze(1).to(torch.bool), 0.0) - - # Mix precision to prevent overflow for fp16 - beta = beta.type(dtype) - - beta = beta.clamp(0, 1) - - prob_check(beta) - - return beta - - -def mass_preservation( - alpha: Tensor, - padding_mask: Optional[Tensor] = None, - left_padding: bool = False -): - """ - Function to compute the mass perservation for alpha. - This means that the residual weights of alpha will be assigned - to the last token. - - Reference: - Monotonic Infinite Lookback Attention for Simultaneous Machine Translation - https://arxiv.org/abs/1906.05218 - - alpha: bsz, tgt_len, src_len - padding_mask: bsz, src_len - left_padding: bool - """ - - prob_check(alpha) - - if padding_mask is not None: - if not left_padding: - assert not padding_mask[:, 0].any(), ( - "Find padding on the beginning of the sequence." - ) - alpha = alpha.masked_fill(padding_mask.unsqueeze(1), 0.0) - - if left_padding or padding_mask is None: - residuals = 1 - alpha[:, :, :-1].sum(dim=-1).clamp(0, 1) - alpha[:, :, -1] = residuals - else: - # right padding - _, tgt_len, src_len = alpha.size() - residuals = 1 - alpha.sum(dim=-1, keepdim=True).clamp(0, 1) - src_lens = src_len - padding_mask.sum(dim=1, keepdim=True) - src_lens = src_lens.expand(-1, tgt_len).contiguous() - # add back the last value - residuals += alpha.gather(2, src_lens.unsqueeze(2) - 1) - alpha = alpha.scatter(2, src_lens.unsqueeze(2) - 1, residuals) - - prob_check(alpha) - - return alpha diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/simultaneous_translation/utils/p_choose_strategy.py b/audio/safeear/safeear_repo/fairseq_ours/examples/simultaneous_translation/utils/p_choose_strategy.py deleted file mode 100644 index e3ca0f0d31848e6b942da74a21ff332d79e7abc9..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/simultaneous_translation/utils/p_choose_strategy.py +++ /dev/null @@ -1,127 +0,0 @@ -from typing import Dict, Optional - -import torch -from torch import Tensor - - -def waitk_p_choose( - tgt_len: int, - src_len: int, - bsz: int, - waitk_lagging: int, - key_padding_mask: Optional[Tensor] = None, - incremental_state: Optional[Dict[str, Dict[str, Optional[Tensor]]]] = None -): - - max_src_len = src_len - if incremental_state is not None: - # Retrieve target length from incremental states - # For inference the length of query is always 1 - max_tgt_len = incremental_state["steps"]["tgt"] - assert max_tgt_len is not None - max_tgt_len = int(max_tgt_len) - else: - max_tgt_len = tgt_len - - if max_src_len < waitk_lagging: - if incremental_state is not None: - max_tgt_len = 1 - return torch.zeros( - bsz, max_tgt_len, max_src_len - ) - - # Assuming the p_choose looks like this for wait k=3 - # src_len = 6, max_tgt_len = 5 - # [0, 0, 1, 0, 0, 0, 0] - # [0, 0, 0, 1, 0, 0, 0] - # [0, 0, 0, 0, 1, 0, 0] - # [0, 0, 0, 0, 0, 1, 0] - # [0, 0, 0, 0, 0, 0, 1] - # linearize the p_choose matrix: - # [0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0...] - # The indices of linearized matrix that equals 1 is - # 2 + 6 * 0 - # 3 + 6 * 1 - # ... - # n + src_len * n + k - 1 = n * (src_len + 1) + k - 1 - # n from 0 to max_tgt_len - 1 - # - # First, generate the indices (activate_indices_offset: bsz, max_tgt_len) - # Second, scatter a zeros tensor (bsz, max_tgt_len * src_len) - # with activate_indices_offset - # Third, resize the tensor to (bsz, max_tgt_len, src_len) - - activate_indices_offset = ( - ( - torch.arange(max_tgt_len) * (max_src_len + 1) - + waitk_lagging - 1 - ) - .unsqueeze(0) - .expand(bsz, max_tgt_len) - .long() - ) - - if key_padding_mask is not None: - if key_padding_mask[:, 0].any(): - # Left padding - activate_indices_offset += ( - key_padding_mask.sum(dim=1, keepdim=True) - ) - - # Need to clamp the indices that are too large - activate_indices_offset = ( - activate_indices_offset - .clamp( - 0, - min( - [ - max_tgt_len, - max_src_len - waitk_lagging + 1 - ] - ) * max_src_len - 1 - ) - ) - - p_choose = torch.zeros(bsz, max_tgt_len * max_src_len) - - p_choose = p_choose.scatter( - 1, - activate_indices_offset, - 1.0 - ).view(bsz, max_tgt_len, max_src_len) - - if key_padding_mask is not None: - p_choose = p_choose.to(key_padding_mask) - p_choose = p_choose.masked_fill(key_padding_mask.unsqueeze(1), 0) - - if incremental_state is not None: - p_choose = p_choose[:, -1:] - - return p_choose.float() - - -def learnable_p_choose( - energy, - noise_mean: float = 0.0, - noise_var: float = 0.0, - training: bool = True -): - """ - Calculating step wise prob for reading and writing - 1 to read, 0 to write - energy: bsz, tgt_len, src_len - """ - - noise = 0 - if training: - # add noise here to encourage discretness - noise = ( - torch.normal(noise_mean, noise_var, energy.size()) - .type_as(energy) - .to(energy.device) - ) - - p_choose = torch.sigmoid(energy + noise) - - # p_choose: bsz * self.num_heads, tgt_len, src_len - return p_choose diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/README.md b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/README.md deleted file mode 100644 index 17030bf0fd50bb843a508e13e97ed436eae33287..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/README.md +++ /dev/null @@ -1,83 +0,0 @@ -### 2021 Update: We are merging this example into the [S2T framework](../speech_to_text), which supports more generic speech-to-text tasks (e.g. speech translation) and more flexible data processing pipelines. Please stay tuned. - -# Speech Recognition -`examples/speech_recognition` is implementing ASR task in Fairseq, along with needed features, datasets, models and loss functions to train and infer model described in [Transformers with convolutional context for ASR (Abdelrahman Mohamed et al., 2019)](https://arxiv.org/abs/1904.11660). - - -## Additional dependencies -On top of main fairseq dependencies there are couple more additional requirements. - -1) Please follow the instructions to install [torchaudio](https://github.com/pytorch/audio). This is required to compute audio fbank features. -2) [Sclite](http://www1.icsi.berkeley.edu/Speech/docs/sctk-1.2/sclite.htm#sclite_name_0) is used to measure WER. Sclite can be downloaded and installed from source from sctk package [here](http://www.openslr.org/4/). Training and inference doesn't require Sclite dependency. -3) [sentencepiece](https://github.com/google/sentencepiece) is required in order to create dataset with word-piece targets. - -## Preparing librispeech data -``` -./examples/speech_recognition/datasets/prepare-librispeech.sh $DIR_TO_SAVE_RAW_DATA $DIR_FOR_PREPROCESSED_DATA -``` - -## Training librispeech data -``` -python train.py $DIR_FOR_PREPROCESSED_DATA --save-dir $MODEL_PATH --max-epoch 80 --task speech_recognition --arch vggtransformer_2 --optimizer adadelta --lr 1.0 --adadelta-eps 1e-8 --adadelta-rho 0.95 --clip-norm 10.0 --max-tokens 5000 --log-format json --log-interval 1 --criterion cross_entropy_acc --user-dir examples/speech_recognition/ -``` - -## Inference for librispeech -`$SET` can be `test_clean` or `test_other` -Any checkpoint in `$MODEL_PATH` can be selected. In this example we are working with `checkpoint_last.pt` -``` -python examples/speech_recognition/infer.py $DIR_FOR_PREPROCESSED_DATA --task speech_recognition --max-tokens 25000 --nbest 1 --path $MODEL_PATH/checkpoint_last.pt --beam 20 --results-path $RES_DIR --batch-size 40 --gen-subset $SET --user-dir examples/speech_recognition/ -``` - -## Inference for librispeech -``` -sclite -r ${RES_DIR}/ref.word-checkpoint_last.pt-${SET}.txt -h ${RES_DIR}/hypo.word-checkpoint_last.pt-${SET}.txt -i rm -o all stdout > $RES_REPORT -``` -`Sum/Avg` row from first table of the report has WER - -## Using flashlight (previously called [wav2letter](https://github.com/facebookresearch/wav2letter)) components -[flashlight](https://github.com/facebookresearch/flashlight) now has integration with fairseq. Currently this includes: - -* AutoSegmentationCriterion (ASG) -* flashlight-style Conv/GLU model -* flashlight's beam search decoder - -To use these, follow the instructions on [this page](https://github.com/facebookresearch/flashlight/tree/master/bindings/python) to install python bindings. - -## Training librispeech data (flashlight style, Conv/GLU + ASG loss) -Training command: -``` -python train.py $DIR_FOR_PREPROCESSED_DATA --save-dir $MODEL_PATH --max-epoch 100 --task speech_recognition --arch w2l_conv_glu_enc --batch-size 4 --optimizer sgd --lr 0.3,0.8 --momentum 0.8 --clip-norm 0.2 --max-tokens 50000 --log-format json --log-interval 100 --num-workers 0 --sentence-avg --criterion asg_loss --asg-transitions-init 5 --max-replabel 2 --linseg-updates 8789 --user-dir examples/speech_recognition -``` - -Note that ASG loss currently doesn't do well with word-pieces. You should prepare a dataset with character targets by setting `nbpe=31` in `prepare-librispeech.sh`. - -## Inference for librispeech (flashlight decoder, n-gram LM) -Inference command: -``` -python examples/speech_recognition/infer.py $DIR_FOR_PREPROCESSED_DATA --task speech_recognition --seed 1 --nbest 1 --path $MODEL_PATH/checkpoint_last.pt --gen-subset $SET --results-path $RES_DIR --w2l-decoder kenlm --kenlm-model $KENLM_MODEL_PATH --lexicon $LEXICON_PATH --beam 200 --beam-threshold 15 --lm-weight 1.5 --word-score 1.5 --sil-weight -0.3 --criterion asg_loss --max-replabel 2 --user-dir examples/speech_recognition -``` - -`$KENLM_MODEL_PATH` should be a standard n-gram language model file. `$LEXICON_PATH` should be a flashlight-style lexicon (list of known words and their spellings). For ASG inference, a lexicon line should look like this (note the repetition labels): -``` -doorbell D O 1 R B E L 1 ▁ -``` -For CTC inference with word-pieces, repetition labels are not used and the lexicon should have most common spellings for each word (one can use sentencepiece's `NBestEncodeAsPieces` for this): -``` -doorbell ▁DOOR BE LL -doorbell ▁DOOR B E LL -doorbell ▁DO OR BE LL -doorbell ▁DOOR B EL L -doorbell ▁DOOR BE L L -doorbell ▁DO OR B E LL -doorbell ▁DOOR B E L L -doorbell ▁DO OR B EL L -doorbell ▁DO O R BE LL -doorbell ▁DO OR BE L L -``` -Lowercase vs. uppercase matters: the *word* should match the case of the n-gram language model (i.e. `$KENLM_MODEL_PATH`), while the *spelling* should match the case of the token dictionary (i.e. `$DIR_FOR_PREPROCESSED_DATA/dict.txt`). - -## Inference for librispeech (flashlight decoder, viterbi only) -Inference command: -``` -python examples/speech_recognition/infer.py $DIR_FOR_PREPROCESSED_DATA --task speech_recognition --seed 1 --nbest 1 --path $MODEL_PATH/checkpoint_last.pt --gen-subset $SET --results-path $RES_DIR --w2l-decoder viterbi --criterion asg_loss --max-replabel 2 --user-dir examples/speech_recognition -``` diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/__init__.py deleted file mode 100644 index 0278f6a27340c7ff7e207d09348483d1b0d3a100..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from . import criterions, models, tasks # noqa diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/criterions/ASG_loss.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/criterions/ASG_loss.py deleted file mode 100644 index 41f50bbd70388ce723f2d316d4e9776bcd6be3c9..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/criterions/ASG_loss.py +++ /dev/null @@ -1,170 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import torch -from examples.speech_recognition.data.replabels import pack_replabels -from fairseq import utils -from fairseq.criterions import FairseqCriterion, register_criterion - - -@register_criterion("asg_loss") -class ASGCriterion(FairseqCriterion): - @staticmethod - def add_args(parser): - group = parser.add_argument_group("ASG Loss") - group.add_argument( - "--asg-transitions-init", - help="initial diagonal value of transition matrix", - type=float, - default=0.0, - ) - group.add_argument( - "--max-replabel", help="maximum # of replabels", type=int, default=2 - ) - group.add_argument( - "--linseg-updates", - help="# of training updates to use LinSeg initialization", - type=int, - default=0, - ) - group.add_argument( - "--hide-linseg-messages", - help="hide messages about LinSeg initialization", - action="store_true", - ) - - def __init__( - self, - task, - silence_token, - asg_transitions_init, - max_replabel, - linseg_updates, - hide_linseg_messages, - ): - from flashlight.lib.sequence.criterion import ASGLoss, CriterionScaleMode - - super().__init__(task) - self.tgt_dict = task.target_dictionary - self.eos = self.tgt_dict.eos() - self.silence = ( - self.tgt_dict.index(silence_token) - if silence_token in self.tgt_dict - else None - ) - self.max_replabel = max_replabel - - num_labels = len(self.tgt_dict) - self.asg = ASGLoss(num_labels, scale_mode=CriterionScaleMode.TARGET_SZ_SQRT) - self.asg.trans = torch.nn.Parameter( - asg_transitions_init * torch.eye(num_labels), requires_grad=True - ) - - self.linseg_progress = torch.nn.Parameter( - torch.tensor([0], dtype=torch.int), requires_grad=False - ) - self.linseg_maximum = linseg_updates - self.linseg_message_state = "none" if hide_linseg_messages else "start" - - @classmethod - def build_criterion(cls, args, task): - return cls( - task, - args.silence_token, - args.asg_transitions_init, - args.max_replabel, - args.linseg_updates, - args.hide_linseg_messages, - ) - - def linseg_step(self): - if not self.training: - return False - if self.linseg_progress.item() < self.linseg_maximum: - if self.linseg_message_state == "start": - print("| using LinSeg to initialize ASG") - self.linseg_message_state = "finish" - self.linseg_progress.add_(1) - return True - elif self.linseg_message_state == "finish": - print("| finished LinSeg initialization") - self.linseg_message_state = "none" - return False - - def replace_eos_with_silence(self, tgt): - if tgt[-1] != self.eos: - return tgt - elif self.silence is None or (len(tgt) > 1 and tgt[-2] == self.silence): - return tgt[:-1] - else: - return tgt[:-1] + [self.silence] - - def forward(self, model, sample, reduce=True): - """Compute the loss for the given sample. - - Returns a tuple with three elements: - 1) the loss - 2) the sample size, which is used as the denominator for the gradient - 3) logging outputs to display while training - """ - - net_output = model(**sample["net_input"]) - emissions = net_output["encoder_out"].transpose(0, 1).contiguous() - B = emissions.size(0) - T = emissions.size(1) - device = emissions.device - - target = torch.IntTensor(B, T) - target_size = torch.IntTensor(B) - using_linseg = self.linseg_step() - - for b in range(B): - initial_target_size = sample["target_lengths"][b].item() - if initial_target_size == 0: - raise ValueError("target size cannot be zero") - - tgt = sample["target"][b, :initial_target_size].tolist() - tgt = self.replace_eos_with_silence(tgt) - tgt = pack_replabels(tgt, self.tgt_dict, self.max_replabel) - tgt = tgt[:T] - - if using_linseg: - tgt = [tgt[t * len(tgt) // T] for t in range(T)] - - target[b][: len(tgt)] = torch.IntTensor(tgt) - target_size[b] = len(tgt) - - loss = self.asg.forward(emissions, target.to(device), target_size.to(device)) - - if reduce: - loss = torch.sum(loss) - - sample_size = ( - sample["target"].size(0) if self.args.sentence_avg else sample["ntokens"] - ) - logging_output = { - "loss": utils.item(loss.data) if reduce else loss.data, - "ntokens": sample["ntokens"], - "nsentences": sample["target"].size(0), - "sample_size": sample_size, - } - return loss, sample_size, logging_output - - @staticmethod - def aggregate_logging_outputs(logging_outputs): - """Aggregate logging outputs from data parallel training.""" - loss_sum = sum(log.get("loss", 0) for log in logging_outputs) - ntokens = sum(log.get("ntokens", 0) for log in logging_outputs) - nsentences = sum(log.get("nsentences", 0) for log in logging_outputs) - sample_size = sum(log.get("sample_size", 0) for log in logging_outputs) - agg_output = { - "loss": loss_sum / nsentences, - "ntokens": ntokens, - "nsentences": nsentences, - "sample_size": sample_size, - } - return agg_output diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/criterions/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/criterions/__init__.py deleted file mode 100644 index 84282d1bf7384692d790c4decdad7ec9276debc9..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/criterions/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -import importlib -import os - -# ASG loss requires flashlight bindings -files_to_skip = set() -try: - import flashlight.lib.sequence.criterion -except ImportError: - files_to_skip.add("ASG_loss.py") - -for file in sorted(os.listdir(os.path.dirname(__file__))): - if file.endswith(".py") and not file.startswith("_") and file not in files_to_skip: - criterion_name = file[: file.find(".py")] - importlib.import_module( - "examples.speech_recognition.criterions." + criterion_name - ) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/criterions/cross_entropy_acc.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/criterions/cross_entropy_acc.py deleted file mode 100644 index 7c4d8ba3802a2da9467c42b0aa18653c7bbb2ec9..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/criterions/cross_entropy_acc.py +++ /dev/null @@ -1,130 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from __future__ import absolute_import, division, print_function, unicode_literals - -import logging -import math - -import torch -import torch.nn.functional as F -from fairseq import utils -from fairseq.criterions import FairseqCriterion, register_criterion - - -@register_criterion("cross_entropy_acc") -class CrossEntropyWithAccCriterion(FairseqCriterion): - def __init__(self, task, sentence_avg): - super().__init__(task) - self.sentence_avg = sentence_avg - - def compute_loss(self, model, net_output, target, reduction, log_probs): - # N, T -> N * T - target = target.view(-1) - lprobs = model.get_normalized_probs(net_output, log_probs=log_probs) - if not hasattr(lprobs, "batch_first"): - logging.warning( - "ERROR: we need to know whether " - "batch first for the net output; " - "you need to set batch_first attribute for the return value of " - "model.get_normalized_probs. Now, we assume this is true, but " - "in the future, we will raise exception instead. " - ) - batch_first = getattr(lprobs, "batch_first", True) - if not batch_first: - lprobs = lprobs.transpose(0, 1) - - # N, T, D -> N * T, D - lprobs = lprobs.view(-1, lprobs.size(-1)) - loss = F.nll_loss( - lprobs, target, ignore_index=self.padding_idx, reduction=reduction - ) - return lprobs, loss - - def get_logging_output(self, sample, target, lprobs, loss): - target = target.view(-1) - mask = target != self.padding_idx - correct = torch.sum( - lprobs.argmax(1).masked_select(mask) == target.masked_select(mask) - ) - total = torch.sum(mask) - sample_size = ( - sample["target"].size(0) if self.sentence_avg else sample["ntokens"] - ) - - logging_output = { - "loss": utils.item(loss.data), # * sample['ntokens'], - "ntokens": sample["ntokens"], - "nsentences": sample["target"].size(0), - "sample_size": sample_size, - "correct": utils.item(correct.data), - "total": utils.item(total.data), - "nframes": torch.sum(sample["net_input"]["src_lengths"]).item(), - } - - return sample_size, logging_output - - def forward(self, model, sample, reduction="sum", log_probs=True): - """Computes the cross entropy with accuracy metric for the given sample. - - This is similar to CrossEntropyCriterion in fairseq, but also - computes accuracy metrics as part of logging - - Args: - logprobs (Torch.tensor) of shape N, T, D i.e. - batchsize, timesteps, dimensions - targets (Torch.tensor) of shape N, T i.e batchsize, timesteps - - Returns: - tuple: With three elements: - 1) the loss - 2) the sample size, which is used as the denominator for the gradient - 3) logging outputs to display while training - - TODO: - * Currently this Criterion will only work with LSTMEncoderModels or - FairseqModels which have decoder, or Models which return TorchTensor - as net_output. - We need to make a change to support all FairseqEncoder models. - """ - net_output = model(**sample["net_input"]) - target = model.get_targets(sample, net_output) - lprobs, loss = self.compute_loss( - model, net_output, target, reduction, log_probs - ) - sample_size, logging_output = self.get_logging_output( - sample, target, lprobs, loss - ) - return loss, sample_size, logging_output - - @staticmethod - def aggregate_logging_outputs(logging_outputs): - """Aggregate logging outputs from data parallel training.""" - correct_sum = sum(log.get("correct", 0) for log in logging_outputs) - total_sum = sum(log.get("total", 0) for log in logging_outputs) - loss_sum = sum(log.get("loss", 0) for log in logging_outputs) - ntokens = sum(log.get("ntokens", 0) for log in logging_outputs) - nsentences = sum(log.get("nsentences", 0) for log in logging_outputs) - sample_size = sum(log.get("sample_size", 0) for log in logging_outputs) - nframes = sum(log.get("nframes", 0) for log in logging_outputs) - agg_output = { - "loss": loss_sum / sample_size / math.log(2) if sample_size > 0 else 0.0, - # if args.sentence_avg, then sample_size is nsentences, then loss - # is per-sentence loss; else sample_size is ntokens, the loss - # becomes per-output token loss - "ntokens": ntokens, - "nsentences": nsentences, - "nframes": nframes, - "sample_size": sample_size, - "acc": correct_sum * 100.0 / total_sum if total_sum > 0 else 0.0, - "correct": correct_sum, - "total": total_sum, - # total is the number of validate tokens - } - if sample_size != ntokens: - agg_output["nll_loss"] = loss_sum / ntokens / math.log(2) - # loss: per output token loss - # nll_loss: per sentence loss - return agg_output diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/data/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/data/__init__.py deleted file mode 100644 index 16fa0ccecd42d99d2a7130eb052b543d4904c820..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/data/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from .asr_dataset import AsrDataset - -__all__ = [ - "AsrDataset", -] diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/data/asr_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/data/asr_dataset.py deleted file mode 100644 index 63a6fcac85d73b1fce8e4d044b4209b1b67fa8ce..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/data/asr_dataset.py +++ /dev/null @@ -1,122 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import os - -import numpy as np -from fairseq.data import FairseqDataset - -from . import data_utils -from .collaters import Seq2SeqCollater - - -class AsrDataset(FairseqDataset): - """ - A dataset representing speech and corresponding transcription. - - Args: - aud_paths: (List[str]): A list of str with paths to audio files. - aud_durations_ms (List[int]): A list of int containing the durations of - audio files. - tgt (List[torch.LongTensor]): A list of LongTensors containing the indices - of target transcriptions. - tgt_dict (~fairseq.data.Dictionary): target vocabulary. - ids (List[str]): A list of utterance IDs. - speakers (List[str]): A list of speakers corresponding to utterances. - num_mel_bins (int): Number of triangular mel-frequency bins (default: 80) - frame_length (float): Frame length in milliseconds (default: 25.0) - frame_shift (float): Frame shift in milliseconds (default: 10.0) - """ - - def __init__( - self, - aud_paths, - aud_durations_ms, - tgt, - tgt_dict, - ids, - speakers, - num_mel_bins=80, - frame_length=25.0, - frame_shift=10.0, - ): - assert frame_length > 0 - assert frame_shift > 0 - assert all(x > frame_length for x in aud_durations_ms) - self.frame_sizes = [ - int(1 + (d - frame_length) / frame_shift) for d in aud_durations_ms - ] - - assert len(aud_paths) > 0 - assert len(aud_paths) == len(aud_durations_ms) - assert len(aud_paths) == len(tgt) - assert len(aud_paths) == len(ids) - assert len(aud_paths) == len(speakers) - self.aud_paths = aud_paths - self.tgt_dict = tgt_dict - self.tgt = tgt - self.ids = ids - self.speakers = speakers - self.num_mel_bins = num_mel_bins - self.frame_length = frame_length - self.frame_shift = frame_shift - - self.s2s_collater = Seq2SeqCollater( - 0, - 1, - pad_index=self.tgt_dict.pad(), - eos_index=self.tgt_dict.eos(), - move_eos_to_beginning=True, - ) - - def __getitem__(self, index): - import torchaudio - import torchaudio.compliance.kaldi as kaldi - - tgt_item = self.tgt[index] if self.tgt is not None else None - - path = self.aud_paths[index] - if not os.path.exists(path): - raise FileNotFoundError("Audio file not found: {}".format(path)) - sound, sample_rate = torchaudio.load_wav(path) - output = kaldi.fbank( - sound, - num_mel_bins=self.num_mel_bins, - frame_length=self.frame_length, - frame_shift=self.frame_shift, - ) - output_cmvn = data_utils.apply_mv_norm(output) - - return {"id": index, "data": [output_cmvn.detach(), tgt_item]} - - def __len__(self): - return len(self.aud_paths) - - def collater(self, samples): - """Merge a list of samples to form a mini-batch. - - Args: - samples (List[int]): sample indices to collate - - Returns: - dict: a mini-batch suitable for forwarding with a Model - """ - return self.s2s_collater.collate(samples) - - def num_tokens(self, index): - return self.frame_sizes[index] - - def size(self, index): - """Return an example's size as a float or tuple. This value is used when - filtering a dataset with ``--max-positions``.""" - return ( - self.frame_sizes[index], - len(self.tgt[index]) if self.tgt is not None else 0, - ) - - def ordered_indices(self): - """Return an ordered list of indices. Batches will be constructed based - on this order.""" - return np.arange(len(self)) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/data/collaters.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/data/collaters.py deleted file mode 100644 index 6acfec876b87e5a00bc92083b1181301a2a18e3f..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/data/collaters.py +++ /dev/null @@ -1,131 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. -""" - This module contains collection of classes which implement - collate functionalities for various tasks. - - Collaters should know what data to expect for each sample - and they should pack / collate them into batches -""" - - -from __future__ import absolute_import, division, print_function, unicode_literals - -import numpy as np -import torch -from fairseq.data import data_utils as fairseq_data_utils - - -class Seq2SeqCollater(object): - """ - Implements collate function mainly for seq2seq tasks - This expects each sample to contain feature (src_tokens) and - targets. - This collator is also used for aligned training task. - """ - - def __init__( - self, - feature_index=0, - label_index=1, - pad_index=1, - eos_index=2, - move_eos_to_beginning=True, - ): - self.feature_index = feature_index - self.label_index = label_index - self.pad_index = pad_index - self.eos_index = eos_index - self.move_eos_to_beginning = move_eos_to_beginning - - def _collate_frames(self, frames): - """Convert a list of 2d frames into a padded 3d tensor - Args: - frames (list): list of 2d frames of size L[i]*f_dim. Where L[i] is - length of i-th frame and f_dim is static dimension of features - Returns: - 3d tensor of size len(frames)*len_max*f_dim where len_max is max of L[i] - """ - len_max = max(frame.size(0) for frame in frames) - f_dim = frames[0].size(1) - res = frames[0].new(len(frames), len_max, f_dim).fill_(0.0) - - for i, v in enumerate(frames): - res[i, : v.size(0)] = v - - return res - - def collate(self, samples): - """ - utility function to collate samples into batch for speech recognition. - """ - if len(samples) == 0: - return {} - - # parse samples into torch tensors - parsed_samples = [] - for s in samples: - # skip invalid samples - if s["data"][self.feature_index] is None: - continue - source = s["data"][self.feature_index] - if isinstance(source, (np.ndarray, np.generic)): - source = torch.from_numpy(source) - target = s["data"][self.label_index] - if isinstance(target, (np.ndarray, np.generic)): - target = torch.from_numpy(target).long() - elif isinstance(target, list): - target = torch.LongTensor(target) - - parsed_sample = {"id": s["id"], "source": source, "target": target} - parsed_samples.append(parsed_sample) - samples = parsed_samples - - id = torch.LongTensor([s["id"] for s in samples]) - frames = self._collate_frames([s["source"] for s in samples]) - # sort samples by descending number of frames - frames_lengths = torch.LongTensor([s["source"].size(0) for s in samples]) - frames_lengths, sort_order = frames_lengths.sort(descending=True) - id = id.index_select(0, sort_order) - frames = frames.index_select(0, sort_order) - - target = None - target_lengths = None - prev_output_tokens = None - if samples[0].get("target", None) is not None: - ntokens = sum(len(s["target"]) for s in samples) - target = fairseq_data_utils.collate_tokens( - [s["target"] for s in samples], - self.pad_index, - self.eos_index, - left_pad=False, - move_eos_to_beginning=False, - ) - target = target.index_select(0, sort_order) - target_lengths = torch.LongTensor( - [s["target"].size(0) for s in samples] - ).index_select(0, sort_order) - prev_output_tokens = fairseq_data_utils.collate_tokens( - [s["target"] for s in samples], - self.pad_index, - self.eos_index, - left_pad=False, - move_eos_to_beginning=self.move_eos_to_beginning, - ) - prev_output_tokens = prev_output_tokens.index_select(0, sort_order) - else: - ntokens = sum(len(s["source"]) for s in samples) - - batch = { - "id": id, - "ntokens": ntokens, - "net_input": {"src_tokens": frames, "src_lengths": frames_lengths}, - "target": target, - "target_lengths": target_lengths, - "nsentences": len(samples), - } - if prev_output_tokens is not None: - batch["net_input"]["prev_output_tokens"] = prev_output_tokens - return batch diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/data/data_utils.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/data/data_utils.py deleted file mode 100644 index cc4729e63c8ef551b29617d1169a44c24f509ad0..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/data/data_utils.py +++ /dev/null @@ -1,100 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import torch - - -def calc_mean_invstddev(feature): - if len(feature.size()) != 2: - raise ValueError("We expect the input feature to be 2-D tensor") - mean = feature.mean(0) - var = feature.var(0) - # avoid division by ~zero - eps = 1e-8 - if (var < eps).any(): - return mean, 1.0 / (torch.sqrt(var) + eps) - return mean, 1.0 / torch.sqrt(var) - - -def apply_mv_norm(features): - # If there is less than 2 spectrograms, the variance cannot be computed (is NaN) - # and normalization is not possible, so return the item as it is - if features.size(0) < 2: - return features - mean, invstddev = calc_mean_invstddev(features) - res = (features - mean) * invstddev - return res - - -def lengths_to_encoder_padding_mask(lengths, batch_first=False): - """ - convert lengths (a 1-D Long/Int tensor) to 2-D binary tensor - - Args: - lengths: a (B, )-shaped tensor - - Return: - max_length: maximum length of B sequences - encoder_padding_mask: a (max_length, B) binary mask, where - [t, b] = 0 for t < lengths[b] and 1 otherwise - - TODO: - kernelize this function if benchmarking shows this function is slow - """ - max_lengths = torch.max(lengths).item() - bsz = lengths.size(0) - encoder_padding_mask = torch.arange( - max_lengths - ).to( # a (T, ) tensor with [0, ..., T-1] - lengths.device - ).view( # move to the right device - 1, max_lengths - ).expand( # reshape to (1, T)-shaped tensor - bsz, -1 - ) >= lengths.view( # expand to (B, T)-shaped tensor - bsz, 1 - ).expand( - -1, max_lengths - ) - if not batch_first: - return encoder_padding_mask.t(), max_lengths - else: - return encoder_padding_mask, max_lengths - - -def encoder_padding_mask_to_lengths( - encoder_padding_mask, max_lengths, batch_size, device -): - """ - convert encoder_padding_mask (2-D binary tensor) to a 1-D tensor - - Conventionally, encoder output contains a encoder_padding_mask, which is - a 2-D mask in a shape (T, B), whose (t, b) element indicate whether - encoder_out[t, b] is a valid output (=0) or not (=1). Occasionally, we - need to convert this mask tensor to a 1-D tensor in shape (B, ), where - [b] denotes the valid length of b-th sequence - - Args: - encoder_padding_mask: a (T, B)-shaped binary tensor or None; if None, - indicating all are valid - Return: - seq_lengths: a (B,)-shaped tensor, where its (b, )-th element is the - number of valid elements of b-th sequence - - max_lengths: maximum length of all sequence, if encoder_padding_mask is - not None, max_lengths must equal to encoder_padding_mask.size(0) - - batch_size: batch size; if encoder_padding_mask is - not None, max_lengths must equal to encoder_padding_mask.size(1) - - device: which device to put the result on - """ - if encoder_padding_mask is None: - return torch.Tensor([max_lengths] * batch_size).to(torch.int32).to(device) - - assert encoder_padding_mask.size(0) == max_lengths, "max_lengths does not match" - assert encoder_padding_mask.size(1) == batch_size, "batch_size does not match" - - return max_lengths - torch.sum(encoder_padding_mask, dim=0) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/data/replabels.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/data/replabels.py deleted file mode 100644 index 441f1bd432b95865fc981c6c695cee299b07ed62..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/data/replabels.py +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -""" -Replabel transforms for use with flashlight's ASG criterion. -""" - - -def replabel_symbol(i): - """ - Replabel symbols used in flashlight, currently just "1", "2", ... - This prevents training with numeral tokens, so this might change in the future - """ - return str(i) - - -def pack_replabels(tokens, dictionary, max_reps): - """ - Pack a token sequence so that repeated symbols are replaced by replabels - """ - if len(tokens) == 0 or max_reps <= 0: - return tokens - - replabel_value_to_idx = [0] * (max_reps + 1) - for i in range(1, max_reps + 1): - replabel_value_to_idx[i] = dictionary.index(replabel_symbol(i)) - - result = [] - prev_token = -1 - num_reps = 0 - for token in tokens: - if token == prev_token and num_reps < max_reps: - num_reps += 1 - else: - if num_reps > 0: - result.append(replabel_value_to_idx[num_reps]) - num_reps = 0 - result.append(token) - prev_token = token - if num_reps > 0: - result.append(replabel_value_to_idx[num_reps]) - return result - - -def unpack_replabels(tokens, dictionary, max_reps): - """ - Unpack a token sequence so that replabels are replaced by repeated symbols - """ - if len(tokens) == 0 or max_reps <= 0: - return tokens - - replabel_idx_to_value = {} - for i in range(1, max_reps + 1): - replabel_idx_to_value[dictionary.index(replabel_symbol(i))] = i - - result = [] - prev_token = -1 - for token in tokens: - try: - for _ in range(replabel_idx_to_value[token]): - result.append(prev_token) - prev_token = -1 - except KeyError: - result.append(token) - prev_token = token - return result diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/datasets/asr_prep_json.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/datasets/asr_prep_json.py deleted file mode 100644 index a409213e77920d985adb066c3542bd87839c2a02..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/datasets/asr_prep_json.py +++ /dev/null @@ -1,124 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from __future__ import absolute_import, division, print_function, unicode_literals - -import argparse -import concurrent.futures -import json -import multiprocessing -import os -from collections import namedtuple -from itertools import chain - -import sentencepiece as spm -from fairseq.data import Dictionary - -MILLISECONDS_TO_SECONDS = 0.001 - - -def process_sample(aud_path, lable, utt_id, sp, tgt_dict): - import torchaudio - - input = {} - output = {} - si, ei = torchaudio.info(aud_path) - input["length_ms"] = int( - si.length / si.channels / si.rate / MILLISECONDS_TO_SECONDS - ) - input["path"] = aud_path - - token = " ".join(sp.EncodeAsPieces(lable)) - ids = tgt_dict.encode_line(token, append_eos=False) - output["text"] = lable - output["token"] = token - output["tokenid"] = ", ".join(map(str, [t.tolist() for t in ids])) - return {utt_id: {"input": input, "output": output}} - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument( - "--audio-dirs", - nargs="+", - default=["-"], - required=True, - help="input directories with audio files", - ) - parser.add_argument( - "--labels", - required=True, - help="aggregated input labels with format per line", - type=argparse.FileType("r", encoding="UTF-8"), - ) - parser.add_argument( - "--spm-model", - required=True, - help="sentencepiece model to use for encoding", - type=argparse.FileType("r", encoding="UTF-8"), - ) - parser.add_argument( - "--dictionary", - required=True, - help="file to load fairseq dictionary from", - type=argparse.FileType("r", encoding="UTF-8"), - ) - parser.add_argument("--audio-format", choices=["flac", "wav"], default="wav") - parser.add_argument( - "--output", - required=True, - type=argparse.FileType("w"), - help="path to save json output", - ) - args = parser.parse_args() - - sp = spm.SentencePieceProcessor() - sp.Load(args.spm_model.name) - - tgt_dict = Dictionary.load(args.dictionary) - - labels = {} - for line in args.labels: - (utt_id, label) = line.split(" ", 1) - labels[utt_id] = label - if len(labels) == 0: - raise Exception("No labels found in ", args.labels_path) - - Sample = namedtuple("Sample", "aud_path utt_id") - samples = [] - for path, _, files in chain.from_iterable( - os.walk(path) for path in args.audio_dirs - ): - for f in files: - if f.endswith(args.audio_format): - if len(os.path.splitext(f)) != 2: - raise Exception("Expect file name. Got: ", f) - utt_id = os.path.splitext(f)[0] - if utt_id not in labels: - continue - samples.append(Sample(os.path.join(path, f), utt_id)) - - utts = {} - num_cpu = multiprocessing.cpu_count() - with concurrent.futures.ThreadPoolExecutor(max_workers=num_cpu) as executor: - future_to_sample = { - executor.submit( - process_sample, s.aud_path, labels[s.utt_id], s.utt_id, sp, tgt_dict - ): s - for s in samples - } - for future in concurrent.futures.as_completed(future_to_sample): - try: - data = future.result() - except Exception as exc: - print("generated an exception: ", exc) - else: - utts.update(data) - json.dump({"utts": utts}, args.output, indent=4) - - -if __name__ == "__main__": - main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/datasets/prepare-librispeech.sh b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/datasets/prepare-librispeech.sh deleted file mode 100644 index 9e9297f08947027685ff508bfa91ff26b0d8ea0c..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/datasets/prepare-librispeech.sh +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -# Prepare librispeech dataset - -base_url=www.openslr.org/resources/12 -train_dir=train_960 - -if [ "$#" -ne 2 ]; then - echo "Usage: $0 " - echo "e.g.: $0 /tmp/librispeech_raw/ ~/data/librispeech_final" - exit 1 -fi - -download_dir=${1%/} -out_dir=${2%/} - -fairseq_root=~/fairseq-py/ -mkdir -p ${out_dir} -cd ${out_dir} || exit - -nbpe=5000 -bpemode=unigram - -if [ ! -d "$fairseq_root" ]; then - echo "$0: Please set correct fairseq_root" - exit 1 -fi - -echo "Data Download" -for part in dev-clean test-clean dev-other test-other train-clean-100 train-clean-360 train-other-500; do - url=$base_url/$part.tar.gz - if ! wget -P $download_dir $url; then - echo "$0: wget failed for $url" - exit 1 - fi - if ! tar -C $download_dir -xvzf $download_dir/$part.tar.gz; then - echo "$0: error un-tarring archive $download_dir/$part.tar.gz" - exit 1 - fi -done - -echo "Merge all train packs into one" -mkdir -p ${download_dir}/LibriSpeech/${train_dir}/ -for part in train-clean-100 train-clean-360 train-other-500; do - mv ${download_dir}/LibriSpeech/${part}/* $download_dir/LibriSpeech/${train_dir}/ -done -echo "Merge train text" -find ${download_dir}/LibriSpeech/${train_dir}/ -name '*.txt' -exec cat {} \; >> ${download_dir}/LibriSpeech/${train_dir}/text - -# Use combined dev-clean and dev-other as validation set -find ${download_dir}/LibriSpeech/dev-clean/ ${download_dir}/LibriSpeech/dev-other/ -name '*.txt' -exec cat {} \; >> ${download_dir}/LibriSpeech/valid_text -find ${download_dir}/LibriSpeech/test-clean/ -name '*.txt' -exec cat {} \; >> ${download_dir}/LibriSpeech/test-clean/text -find ${download_dir}/LibriSpeech/test-other/ -name '*.txt' -exec cat {} \; >> ${download_dir}/LibriSpeech/test-other/text - - -dict=data/lang_char/${train_dir}_${bpemode}${nbpe}_units.txt -encoded=data/lang_char/${train_dir}_${bpemode}${nbpe}_encoded.txt -fairseq_dict=data/lang_char/${train_dir}_${bpemode}${nbpe}_fairseq_dict.txt -bpemodel=data/lang_char/${train_dir}_${bpemode}${nbpe} -echo "dictionary: ${dict}" -echo "Dictionary preparation" -mkdir -p data/lang_char/ -echo " 3" > ${dict} -echo " 2" >> ${dict} -echo " 1" >> ${dict} -cut -f 2- -d" " ${download_dir}/LibriSpeech/${train_dir}/text > data/lang_char/input.txt -spm_train --input=data/lang_char/input.txt --vocab_size=${nbpe} --model_type=${bpemode} --model_prefix=${bpemodel} --input_sentence_size=100000000 --unk_id=3 --eos_id=2 --pad_id=1 --bos_id=-1 --character_coverage=1 -spm_encode --model=${bpemodel}.model --output_format=piece < data/lang_char/input.txt > ${encoded} -cat ${encoded} | tr ' ' '\n' | sort | uniq | awk '{print $0 " " NR+3}' >> ${dict} -cat ${encoded} | tr ' ' '\n' | sort | uniq -c | awk '{print $2 " " $1}' > ${fairseq_dict} -wc -l ${dict} - -echo "Prepare train and test jsons" -for part in train_960 test-other test-clean; do - python ${fairseq_root}/examples/speech_recognition/datasets/asr_prep_json.py --audio-dirs ${download_dir}/LibriSpeech/${part} --labels ${download_dir}/LibriSpeech/${part}/text --spm-model ${bpemodel}.model --audio-format flac --dictionary ${fairseq_dict} --output ${part}.json -done -# fairseq expects to find train.json and valid.json during training -mv train_960.json train.json - -echo "Prepare valid json" -python ${fairseq_root}/examples/speech_recognition/datasets/asr_prep_json.py --audio-dirs ${download_dir}/LibriSpeech/dev-clean ${download_dir}/LibriSpeech/dev-other --labels ${download_dir}/LibriSpeech/valid_text --spm-model ${bpemodel}.model --audio-format flac --dictionary ${fairseq_dict} --output valid.json - -cp ${fairseq_dict} ./dict.txt -cp ${bpemodel}.model ./spm.model diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/infer.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/infer.py deleted file mode 100644 index 489051ce079e7c27bab17118d57918ec18fec850..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/infer.py +++ /dev/null @@ -1,435 +0,0 @@ -#!/usr/bin/env python3 -u -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -""" -Run inference for pre-processed data with a trained model. -""" - -import ast -import logging -import math -import os -import sys - -import editdistance -import numpy as np -import torch -from fairseq import checkpoint_utils, options, progress_bar, tasks, utils -from fairseq.data.data_utils import post_process -from fairseq.logging.meters import StopwatchMeter, TimeMeter - -logging.basicConfig() -logging.root.setLevel(logging.INFO) -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - - -def add_asr_eval_argument(parser): - parser.add_argument("--kspmodel", default=None, help="sentence piece model") - parser.add_argument( - "--wfstlm", default=None, help="wfstlm on dictonary output units" - ) - parser.add_argument( - "--rnnt_decoding_type", - default="greedy", - help="wfstlm on dictonary\ -output units", - ) - try: - parser.add_argument( - "--lm-weight", - "--lm_weight", - type=float, - default=0.2, - help="weight for lm while interpolating with neural score", - ) - except: - pass - parser.add_argument( - "--rnnt_len_penalty", default=-0.5, help="rnnt length penalty on word level" - ) - parser.add_argument( - "--w2l-decoder", - choices=["viterbi", "kenlm", "fairseqlm"], - help="use a w2l decoder", - ) - parser.add_argument("--lexicon", help="lexicon for w2l decoder") - parser.add_argument("--unit-lm", action="store_true", help="if using a unit lm") - parser.add_argument("--kenlm-model", "--lm-model", help="lm model for w2l decoder") - parser.add_argument("--beam-threshold", type=float, default=25.0) - parser.add_argument("--beam-size-token", type=float, default=100) - parser.add_argument("--word-score", type=float, default=1.0) - parser.add_argument("--unk-weight", type=float, default=-math.inf) - parser.add_argument("--sil-weight", type=float, default=0.0) - parser.add_argument( - "--dump-emissions", - type=str, - default=None, - help="if present, dumps emissions into this file and exits", - ) - parser.add_argument( - "--dump-features", - type=str, - default=None, - help="if present, dumps features into this file and exits", - ) - parser.add_argument( - "--load-emissions", - type=str, - default=None, - help="if present, loads emissions from this file", - ) - return parser - - -def check_args(args): - # assert args.path is not None, "--path required for generation!" - # assert args.results_path is not None, "--results_path required for generation!" - assert ( - not args.sampling or args.nbest == args.beam - ), "--sampling requires --nbest to be equal to --beam" - assert ( - args.replace_unk is None or args.raw_text - ), "--replace-unk requires a raw text dataset (--raw-text)" - - -def get_dataset_itr(args, task, models): - return task.get_batch_iterator( - dataset=task.dataset(args.gen_subset), - max_tokens=args.max_tokens, - max_sentences=args.batch_size, - max_positions=(sys.maxsize, sys.maxsize), - ignore_invalid_inputs=args.skip_invalid_size_inputs_valid_test, - required_batch_size_multiple=args.required_batch_size_multiple, - num_shards=args.num_shards, - shard_id=args.shard_id, - num_workers=args.num_workers, - data_buffer_size=args.data_buffer_size, - ).next_epoch_itr(shuffle=False) - - -def process_predictions( - args, hypos, sp, tgt_dict, target_tokens, res_files, speaker, id -): - for hypo in hypos[: min(len(hypos), args.nbest)]: - hyp_pieces = tgt_dict.string(hypo["tokens"].int().cpu()) - - if "words" in hypo: - hyp_words = " ".join(hypo["words"]) - else: - hyp_words = post_process(hyp_pieces, args.post_process) - - if res_files is not None: - print( - "{} ({}-{})".format(hyp_pieces, speaker, id), - file=res_files["hypo.units"], - ) - print( - "{} ({}-{})".format(hyp_words, speaker, id), - file=res_files["hypo.words"], - ) - - tgt_pieces = tgt_dict.string(target_tokens) - tgt_words = post_process(tgt_pieces, args.post_process) - - if res_files is not None: - print( - "{} ({}-{})".format(tgt_pieces, speaker, id), - file=res_files["ref.units"], - ) - print( - "{} ({}-{})".format(tgt_words, speaker, id), file=res_files["ref.words"] - ) - - if not args.quiet: - logger.info("HYPO:" + hyp_words) - logger.info("TARGET:" + tgt_words) - logger.info("___________________") - - hyp_words = hyp_words.split() - tgt_words = tgt_words.split() - return editdistance.eval(hyp_words, tgt_words), len(tgt_words) - - -def prepare_result_files(args): - def get_res_file(file_prefix): - if args.num_shards > 1: - file_prefix = f"{args.shard_id}_{file_prefix}" - path = os.path.join( - args.results_path, - "{}-{}-{}.txt".format( - file_prefix, os.path.basename(args.path), args.gen_subset - ), - ) - return open(path, "w", buffering=1) - - if not args.results_path: - return None - - return { - "hypo.words": get_res_file("hypo.word"), - "hypo.units": get_res_file("hypo.units"), - "ref.words": get_res_file("ref.word"), - "ref.units": get_res_file("ref.units"), - } - - -def optimize_models(args, use_cuda, models): - """Optimize ensemble for generation""" - for model in models: - model.make_generation_fast_( - beamable_mm_beam_size=None if args.no_beamable_mm else args.beam, - need_attn=args.print_alignment, - ) - if args.fp16: - model.half() - if use_cuda: - model.cuda() - - -def apply_half(t): - if t.dtype is torch.float32: - return t.to(dtype=torch.half) - return t - - -class ExistingEmissionsDecoder(object): - def __init__(self, decoder, emissions): - self.decoder = decoder - self.emissions = emissions - - def generate(self, models, sample, **unused): - ids = sample["id"].cpu().numpy() - try: - emissions = np.stack(self.emissions[ids]) - except: - print([x.shape for x in self.emissions[ids]]) - raise Exception("invalid sizes") - emissions = torch.from_numpy(emissions) - return self.decoder.decode(emissions) - - -def main(args, task=None, model_state=None): - check_args(args) - - use_fp16 = args.fp16 - if args.max_tokens is None and args.batch_size is None: - args.max_tokens = 4000000 - logger.info(args) - - use_cuda = torch.cuda.is_available() and not args.cpu - - logger.info("| decoding with criterion {}".format(args.criterion)) - - task = tasks.setup_task(args) - - # Load ensemble - if args.load_emissions: - models, criterions = [], [] - task.load_dataset(args.gen_subset) - else: - logger.info("| loading model(s) from {}".format(args.path)) - models, saved_cfg, task = checkpoint_utils.load_model_ensemble_and_task( - utils.split_paths(args.path, separator="\\"), - arg_overrides=ast.literal_eval(args.model_overrides), - task=task, - suffix=args.checkpoint_suffix, - strict=(args.checkpoint_shard_count == 1), - num_shards=args.checkpoint_shard_count, - state=model_state, - ) - optimize_models(args, use_cuda, models) - task.load_dataset(args.gen_subset, task_cfg=saved_cfg.task) - - - # Set dictionary - tgt_dict = task.target_dictionary - - logger.info( - "| {} {} {} examples".format( - args.data, args.gen_subset, len(task.dataset(args.gen_subset)) - ) - ) - - # hack to pass transitions to W2lDecoder - if args.criterion == "asg_loss": - raise NotImplementedError("asg_loss is currently not supported") - # trans = criterions[0].asg.trans.data - # args.asg_transitions = torch.flatten(trans).tolist() - - # Load dataset (possibly sharded) - itr = get_dataset_itr(args, task, models) - - # Initialize generator - gen_timer = StopwatchMeter() - - def build_generator(args): - w2l_decoder = getattr(args, "w2l_decoder", None) - if w2l_decoder == "viterbi": - from examples.speech_recognition.w2l_decoder import W2lViterbiDecoder - - return W2lViterbiDecoder(args, task.target_dictionary) - elif w2l_decoder == "kenlm": - from examples.speech_recognition.w2l_decoder import W2lKenLMDecoder - - return W2lKenLMDecoder(args, task.target_dictionary) - elif w2l_decoder == "fairseqlm": - from examples.speech_recognition.w2l_decoder import W2lFairseqLMDecoder - - return W2lFairseqLMDecoder(args, task.target_dictionary) - else: - print( - "only flashlight decoders with (viterbi, kenlm, fairseqlm) options are supported at the moment" - ) - - # please do not touch this unless you test both generate.py and infer.py with audio_pretraining task - generator = build_generator(args) - - if args.load_emissions: - generator = ExistingEmissionsDecoder( - generator, np.load(args.load_emissions, allow_pickle=True) - ) - logger.info("loaded emissions from " + args.load_emissions) - - num_sentences = 0 - - if args.results_path is not None and not os.path.exists(args.results_path): - os.makedirs(args.results_path) - - max_source_pos = ( - utils.resolve_max_positions( - task.max_positions(), *[model.max_positions() for model in models] - ), - ) - - if max_source_pos is not None: - max_source_pos = max_source_pos[0] - if max_source_pos is not None: - max_source_pos = max_source_pos[0] - 1 - - if args.dump_emissions: - emissions = {} - if args.dump_features: - features = {} - models[0].bert.proj = None - else: - res_files = prepare_result_files(args) - errs_t = 0 - lengths_t = 0 - with progress_bar.build_progress_bar(args, itr) as t: - wps_meter = TimeMeter() - for sample in t: - sample = utils.move_to_cuda(sample) if use_cuda else sample - if use_fp16: - sample = utils.apply_to_sample(apply_half, sample) - if "net_input" not in sample: - continue - - prefix_tokens = None - if args.prefix_size > 0: - prefix_tokens = sample["target"][:, : args.prefix_size] - - gen_timer.start() - if args.dump_emissions: - with torch.no_grad(): - encoder_out = models[0](**sample["net_input"]) - emm = models[0].get_normalized_probs(encoder_out, log_probs=True) - emm = emm.transpose(0, 1).cpu().numpy() - for i, id in enumerate(sample["id"]): - emissions[id.item()] = emm[i] - continue - elif args.dump_features: - with torch.no_grad(): - encoder_out = models[0](**sample["net_input"]) - feat = encoder_out["encoder_out"].transpose(0, 1).cpu().numpy() - for i, id in enumerate(sample["id"]): - padding = ( - encoder_out["encoder_padding_mask"][i].cpu().numpy() - if encoder_out["encoder_padding_mask"] is not None - else None - ) - features[id.item()] = (feat[i], padding) - continue - hypos = task.inference_step(generator, models, sample, prefix_tokens) - num_generated_tokens = sum(len(h[0]["tokens"]) for h in hypos) - gen_timer.stop(num_generated_tokens) - - for i, sample_id in enumerate(sample["id"].tolist()): - speaker = None - # id = task.dataset(args.gen_subset).ids[int(sample_id)] - id = sample_id - toks = ( - sample["target"][i, :] - if "target_label" not in sample - else sample["target_label"][i, :] - ) - target_tokens = utils.strip_pad(toks, tgt_dict.pad()).int().cpu() - # Process top predictions - errs, length = process_predictions( - args, - hypos[i], - None, - tgt_dict, - target_tokens, - res_files, - speaker, - id, - ) - errs_t += errs - lengths_t += length - - wps_meter.update(num_generated_tokens) - t.log({"wps": round(wps_meter.avg)}) - num_sentences += ( - sample["nsentences"] if "nsentences" in sample else sample["id"].numel() - ) - - wer = None - if args.dump_emissions: - emm_arr = [] - for i in range(len(emissions)): - emm_arr.append(emissions[i]) - np.save(args.dump_emissions, emm_arr) - logger.info(f"saved {len(emissions)} emissions to {args.dump_emissions}") - elif args.dump_features: - feat_arr = [] - for i in range(len(features)): - feat_arr.append(features[i]) - np.save(args.dump_features, feat_arr) - logger.info(f"saved {len(features)} emissions to {args.dump_features}") - else: - if lengths_t > 0: - wer = errs_t * 100.0 / lengths_t - logger.info(f"WER: {wer}") - - logger.info( - "| Processed {} sentences ({} tokens) in {:.1f}s ({:.2f}" - "sentences/s, {:.2f} tokens/s)".format( - num_sentences, - gen_timer.n, - gen_timer.sum, - num_sentences / gen_timer.sum, - 1.0 / gen_timer.avg, - ) - ) - logger.info("| Generate {} with beam={}".format(args.gen_subset, args.beam)) - return task, wer - - -def make_parser(): - parser = options.get_generation_parser() - parser = add_asr_eval_argument(parser) - return parser - - -def cli_main(): - parser = make_parser() - args = options.parse_args_and_arch(parser) - main(args) - - -if __name__ == "__main__": - cli_main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/kaldi/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/kaldi/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/kaldi/add-self-loop-simple.cc b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/kaldi/add-self-loop-simple.cc deleted file mode 100644 index e18fb62df52ab85d7802615d8619b0fd94a08f8c..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/kaldi/add-self-loop-simple.cc +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright (c) Facebook, Inc. and its affiliates. - * - * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. - */ - -#include -#include "fstext/fstext-lib.h" // @manual -#include "util/common-utils.h" // @manual - -/* - * This program is to modify a FST without self-loop by: - * for each incoming arc with non-eps input symbol, add a self-loop arc - * with that non-eps symbol as input and eps as output. - * - * This is to make sure the resultant FST can do deduplication for repeated - * symbols, which is very common in acoustic model - * - */ -namespace { -int32 AddSelfLoopsSimple(fst::StdVectorFst* fst) { - typedef fst::MutableArcIterator IterType; - - int32 num_states_before = fst->NumStates(); - fst::MakePrecedingInputSymbolsSame(false, fst); - int32 num_states_after = fst->NumStates(); - KALDI_LOG << "There are " << num_states_before - << " states in the original FST; " - << " after MakePrecedingInputSymbolsSame, there are " - << num_states_after << " states " << std::endl; - - auto weight_one = fst::StdArc::Weight::One(); - - int32 num_arc_added = 0; - - fst::StdArc self_loop_arc; - self_loop_arc.weight = weight_one; - - int32 num_states = fst->NumStates(); - std::vector> incoming_non_eps_label_per_state(num_states); - - for (int32 state = 0; state < num_states; state++) { - for (IterType aiter(fst, state); !aiter.Done(); aiter.Next()) { - fst::StdArc arc(aiter.Value()); - if (arc.ilabel != 0) { - incoming_non_eps_label_per_state[arc.nextstate].insert(arc.ilabel); - } - } - } - - for (int32 state = 0; state < num_states; state++) { - if (!incoming_non_eps_label_per_state[state].empty()) { - auto& ilabel_set = incoming_non_eps_label_per_state[state]; - for (auto it = ilabel_set.begin(); it != ilabel_set.end(); it++) { - self_loop_arc.ilabel = *it; - self_loop_arc.olabel = 0; - self_loop_arc.nextstate = state; - fst->AddArc(state, self_loop_arc); - num_arc_added++; - } - } - } - return num_arc_added; -} - -void print_usage() { - std::cout << "add-self-loop-simple usage:\n" - "\tadd-self-loop-simple \n"; -} -} // namespace - -int main(int argc, char** argv) { - if (argc != 3) { - print_usage(); - exit(1); - } - - auto input = argv[1]; - auto output = argv[2]; - - auto fst = fst::ReadFstKaldi(input); - auto num_states = fst->NumStates(); - KALDI_LOG << "Loading FST from " << input << " with " << num_states - << " states." << std::endl; - - int32 num_arc_added = AddSelfLoopsSimple(fst); - KALDI_LOG << "Adding " << num_arc_added << " self-loop arcs " << std::endl; - - fst::WriteFstKaldi(*fst, std::string(output)); - KALDI_LOG << "Writing FST to " << output << std::endl; - - delete fst; -} diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/kaldi/config/kaldi_initializer.yaml b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/kaldi/config/kaldi_initializer.yaml deleted file mode 100644 index be9ba98f55463d41d5d5ea35e306abc0886dbead..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/kaldi/config/kaldi_initializer.yaml +++ /dev/null @@ -1,8 +0,0 @@ -# @package _group_ - -data_dir: ??? -fst_dir: ??? -in_labels: ??? -kaldi_root: ??? -lm_arpa: ??? -blank_symbol: diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/kaldi/kaldi_decoder.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/kaldi/kaldi_decoder.py deleted file mode 100644 index 01efcbcdedef7846a1cb6b0c3cc9c325114b0bfa..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/kaldi/kaldi_decoder.py +++ /dev/null @@ -1,243 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import logging -import os -import warnings -from concurrent.futures import ThreadPoolExecutor -from dataclasses import dataclass -from typing import Optional - -import torch -from fairseq.dataclass import FairseqDataclass -from omegaconf import MISSING - -from .kaldi_initializer import KaldiInitializerConfig, initalize_kaldi - -logger = logging.getLogger(__name__) - - -@dataclass -class KaldiDecoderConfig(FairseqDataclass): - hlg_graph_path: Optional[str] = None - output_dict: str = MISSING - - kaldi_initializer_config: Optional[KaldiInitializerConfig] = None - - acoustic_scale: float = 0.5 - max_active: int = 10000 - beam_delta: float = 0.5 - hash_ratio: float = 2.0 - - is_lattice: bool = False - lattice_beam: float = 10.0 - prune_interval: int = 25 - determinize_lattice: bool = True - prune_scale: float = 0.1 - max_mem: int = 0 - phone_determinize: bool = True - word_determinize: bool = True - minimize: bool = True - - num_threads: int = 1 - - -class KaldiDecoder(object): - def __init__( - self, - cfg: KaldiDecoderConfig, - beam: int, - nbest: int = 1, - ): - try: - from kaldi.asr import FasterRecognizer, LatticeFasterRecognizer - from kaldi.base import set_verbose_level - from kaldi.decoder import ( - FasterDecoder, - FasterDecoderOptions, - LatticeFasterDecoder, - LatticeFasterDecoderOptions, - ) - from kaldi.fstext import SymbolTable, read_fst_kaldi - from kaldi.lat.functions import DeterminizeLatticePhonePrunedOptions - except: - warnings.warn( - "pykaldi is required for this functionality. Please install from https://github.com/pykaldi/pykaldi" - ) - - # set_verbose_level(2) - - self.acoustic_scale = cfg.acoustic_scale - self.nbest = nbest - - if cfg.hlg_graph_path is None: - assert ( - cfg.kaldi_initializer_config is not None - ), "Must provide hlg graph path or kaldi initializer config" - cfg.hlg_graph_path = initalize_kaldi(cfg.kaldi_initializer_config) - - assert os.path.exists(cfg.hlg_graph_path), cfg.hlg_graph_path - - if cfg.is_lattice: - self.dec_cls = LatticeFasterDecoder - opt_cls = LatticeFasterDecoderOptions - self.rec_cls = LatticeFasterRecognizer - else: - assert self.nbest == 1, "nbest > 1 requires lattice decoder" - self.dec_cls = FasterDecoder - opt_cls = FasterDecoderOptions - self.rec_cls = FasterRecognizer - - self.decoder_options = opt_cls() - self.decoder_options.beam = beam - self.decoder_options.max_active = cfg.max_active - self.decoder_options.beam_delta = cfg.beam_delta - self.decoder_options.hash_ratio = cfg.hash_ratio - - if cfg.is_lattice: - self.decoder_options.lattice_beam = cfg.lattice_beam - self.decoder_options.prune_interval = cfg.prune_interval - self.decoder_options.determinize_lattice = cfg.determinize_lattice - self.decoder_options.prune_scale = cfg.prune_scale - det_opts = DeterminizeLatticePhonePrunedOptions() - det_opts.max_mem = cfg.max_mem - det_opts.phone_determinize = cfg.phone_determinize - det_opts.word_determinize = cfg.word_determinize - det_opts.minimize = cfg.minimize - self.decoder_options.det_opts = det_opts - - self.output_symbols = {} - with open(cfg.output_dict, "r") as f: - for line in f: - items = line.rstrip().split() - assert len(items) == 2 - self.output_symbols[int(items[1])] = items[0] - - logger.info(f"Loading FST from {cfg.hlg_graph_path}") - self.fst = read_fst_kaldi(cfg.hlg_graph_path) - self.symbol_table = SymbolTable.read_text(cfg.output_dict) - - self.executor = ThreadPoolExecutor(max_workers=cfg.num_threads) - - def generate(self, models, sample, **unused): - """Generate a batch of inferences.""" - # model.forward normally channels prev_output_tokens into the decoder - # separately, but SequenceGenerator directly calls model.encoder - encoder_input = { - k: v for k, v in sample["net_input"].items() if k != "prev_output_tokens" - } - emissions, padding = self.get_emissions(models, encoder_input) - return self.decode(emissions, padding) - - def get_emissions(self, models, encoder_input): - """Run encoder and normalize emissions""" - model = models[0] - - all_encoder_out = [m(**encoder_input) for m in models] - - if len(all_encoder_out) > 1: - - if "encoder_out" in all_encoder_out[0]: - encoder_out = { - "encoder_out": sum(e["encoder_out"] for e in all_encoder_out) - / len(all_encoder_out), - "encoder_padding_mask": all_encoder_out[0]["encoder_padding_mask"], - } - padding = encoder_out["encoder_padding_mask"] - else: - encoder_out = { - "logits": sum(e["logits"] for e in all_encoder_out) - / len(all_encoder_out), - "padding_mask": all_encoder_out[0]["padding_mask"], - } - padding = encoder_out["padding_mask"] - else: - encoder_out = all_encoder_out[0] - padding = ( - encoder_out["padding_mask"] - if "padding_mask" in encoder_out - else encoder_out["encoder_padding_mask"] - ) - - if hasattr(model, "get_logits"): - emissions = model.get_logits(encoder_out, normalize=True) - else: - emissions = model.get_normalized_probs(encoder_out, log_probs=True) - - return ( - emissions.cpu().float().transpose(0, 1), - padding.cpu() if padding is not None and padding.any() else None, - ) - - def decode_one(self, logits, padding): - from kaldi.matrix import Matrix - - decoder = self.dec_cls(self.fst, self.decoder_options) - asr = self.rec_cls( - decoder, self.symbol_table, acoustic_scale=self.acoustic_scale - ) - - if padding is not None: - logits = logits[~padding] - - mat = Matrix(logits.numpy()) - - out = asr.decode(mat) - - if self.nbest > 1: - from kaldi.fstext import shortestpath - from kaldi.fstext.utils import ( - convert_compact_lattice_to_lattice, - convert_lattice_to_std, - convert_nbest_to_list, - get_linear_symbol_sequence, - ) - - lat = out["lattice"] - - sp = shortestpath(lat, nshortest=self.nbest) - - sp = convert_compact_lattice_to_lattice(sp) - sp = convert_lattice_to_std(sp) - seq = convert_nbest_to_list(sp) - - results = [] - for s in seq: - _, o, w = get_linear_symbol_sequence(s) - words = list(self.output_symbols[z] for z in o) - results.append( - { - "tokens": words, - "words": words, - "score": w.value, - "emissions": logits, - } - ) - return results - else: - words = out["text"].split() - return [ - { - "tokens": words, - "words": words, - "score": out["likelihood"], - "emissions": logits, - } - ] - - def decode(self, emissions, padding): - if padding is None: - padding = [None] * len(emissions) - - ret = list( - map( - lambda e, p: self.executor.submit(self.decode_one, e, p), - emissions, - padding, - ) - ) - return ret diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/kaldi/kaldi_initializer.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/kaldi/kaldi_initializer.py deleted file mode 100644 index 717b9190517c8f21633d58b722328c292d66b9ac..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/kaldi/kaldi_initializer.py +++ /dev/null @@ -1,698 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import logging -import os -import os.path as osp -import subprocess -from dataclasses import dataclass -from pathlib import Path -from typing import Optional - -import hydra -from fairseq.data.dictionary import Dictionary -from fairseq.dataclass import FairseqDataclass -from hydra.core.config_store import ConfigStore -from omegaconf import MISSING, OmegaConf - -script_dir = Path(__file__).resolve().parent -config_path = script_dir / "config" - - -logger = logging.getLogger(__name__) - - -@dataclass -class KaldiInitializerConfig(FairseqDataclass): - data_dir: str = MISSING - fst_dir: Optional[str] = None - in_labels: str = MISSING - out_labels: Optional[str] = None - wav2letter_lexicon: Optional[str] = None - lm_arpa: str = MISSING - kaldi_root: str = MISSING - blank_symbol: str = "" - silence_symbol: Optional[str] = None - - -def create_units(fst_dir: Path, in_labels: str, vocab: Dictionary) -> Path: - in_units_file = fst_dir / f"kaldi_dict.{in_labels}.txt" - if not in_units_file.exists(): - - logger.info(f"Creating {in_units_file}") - - with open(in_units_file, "w") as f: - print(" 0", file=f) - i = 1 - for symb in vocab.symbols[vocab.nspecial :]: - if not symb.startswith("madeupword"): - print(f"{symb} {i}", file=f) - i += 1 - return in_units_file - - -def create_lexicon( - cfg: KaldiInitializerConfig, - fst_dir: Path, - unique_label: str, - in_units_file: Path, - out_words_file: Path, -) -> (Path, Path): - - disambig_in_units_file = fst_dir / f"kaldi_dict.{cfg.in_labels}_disambig.txt" - lexicon_file = fst_dir / f"kaldi_lexicon.{unique_label}.txt" - disambig_lexicon_file = fst_dir / f"kaldi_lexicon.{unique_label}_disambig.txt" - if ( - not lexicon_file.exists() - or not disambig_lexicon_file.exists() - or not disambig_in_units_file.exists() - ): - logger.info(f"Creating {lexicon_file} (in units file: {in_units_file})") - - assert cfg.wav2letter_lexicon is not None or cfg.in_labels == cfg.out_labels - - if cfg.wav2letter_lexicon is not None: - lm_words = set() - with open(out_words_file, "r") as lm_dict_f: - for line in lm_dict_f: - lm_words.add(line.split()[0]) - - num_skipped = 0 - total = 0 - with open(cfg.wav2letter_lexicon, "r") as w2l_lex_f, open( - lexicon_file, "w" - ) as out_f: - for line in w2l_lex_f: - items = line.rstrip().split("\t") - assert len(items) == 2, items - if items[0] in lm_words: - print(items[0], items[1], file=out_f) - else: - num_skipped += 1 - logger.debug( - f"Skipping word {items[0]} as it was not found in LM" - ) - total += 1 - if num_skipped > 0: - logger.warning( - f"Skipped {num_skipped} out of {total} words as they were not found in LM" - ) - else: - with open(in_units_file, "r") as in_f, open(lexicon_file, "w") as out_f: - for line in in_f: - symb = line.split()[0] - if symb != "" and symb != "" and symb != "": - print(symb, symb, file=out_f) - - lex_disambig_path = ( - Path(cfg.kaldi_root) / "egs/wsj/s5/utils/add_lex_disambig.pl" - ) - res = subprocess.run( - [lex_disambig_path, lexicon_file, disambig_lexicon_file], - check=True, - capture_output=True, - ) - ndisambig = int(res.stdout) - disamib_path = Path(cfg.kaldi_root) / "egs/wsj/s5/utils/add_disambig.pl" - res = subprocess.run( - [disamib_path, "--include-zero", in_units_file, str(ndisambig)], - check=True, - capture_output=True, - ) - with open(disambig_in_units_file, "wb") as f: - f.write(res.stdout) - - return disambig_lexicon_file, disambig_in_units_file - - -def create_G( - kaldi_root: Path, fst_dir: Path, lm_arpa: Path, arpa_base: str -) -> (Path, Path): - - out_words_file = fst_dir / f"kaldi_dict.{arpa_base}.txt" - grammar_graph = fst_dir / f"G_{arpa_base}.fst" - if not grammar_graph.exists() or not out_words_file.exists(): - logger.info(f"Creating {grammar_graph}") - arpa2fst = kaldi_root / "src/lmbin/arpa2fst" - subprocess.run( - [ - arpa2fst, - "--disambig-symbol=#0", - f"--write-symbol-table={out_words_file}", - lm_arpa, - grammar_graph, - ], - check=True, - ) - return grammar_graph, out_words_file - - -def create_L( - kaldi_root: Path, - fst_dir: Path, - unique_label: str, - lexicon_file: Path, - in_units_file: Path, - out_words_file: Path, -) -> Path: - lexicon_graph = fst_dir / f"L.{unique_label}.fst" - - if not lexicon_graph.exists(): - logger.info(f"Creating {lexicon_graph} (in units: {in_units_file})") - make_lex = kaldi_root / "egs/wsj/s5/utils/make_lexicon_fst.pl" - fstcompile = kaldi_root / "tools/openfst-1.6.7/bin/fstcompile" - fstaddselfloops = kaldi_root / "src/fstbin/fstaddselfloops" - fstarcsort = kaldi_root / "tools/openfst-1.6.7/bin/fstarcsort" - - def write_disambig_symbol(file): - with open(file, "r") as f: - for line in f: - items = line.rstrip().split() - if items[0] == "#0": - out_path = str(file) + "_disamig" - with open(out_path, "w") as out_f: - print(items[1], file=out_f) - return out_path - - return None - - in_disambig_sym = write_disambig_symbol(in_units_file) - assert in_disambig_sym is not None - out_disambig_sym = write_disambig_symbol(out_words_file) - assert out_disambig_sym is not None - - try: - with open(lexicon_graph, "wb") as out_f: - res = subprocess.run( - [make_lex, lexicon_file], capture_output=True, check=True - ) - assert len(res.stderr) == 0, res.stderr.decode("utf-8") - res = subprocess.run( - [ - fstcompile, - f"--isymbols={in_units_file}", - f"--osymbols={out_words_file}", - "--keep_isymbols=false", - "--keep_osymbols=false", - ], - input=res.stdout, - capture_output=True, - ) - assert len(res.stderr) == 0, res.stderr.decode("utf-8") - res = subprocess.run( - [fstaddselfloops, in_disambig_sym, out_disambig_sym], - input=res.stdout, - capture_output=True, - check=True, - ) - res = subprocess.run( - [fstarcsort, "--sort_type=olabel"], - input=res.stdout, - capture_output=True, - check=True, - ) - out_f.write(res.stdout) - except subprocess.CalledProcessError as e: - logger.error(f"cmd: {e.cmd}, err: {e.stderr.decode('utf-8')}") - os.remove(lexicon_graph) - raise - except AssertionError: - os.remove(lexicon_graph) - raise - - return lexicon_graph - - -def create_LG( - kaldi_root: Path, - fst_dir: Path, - unique_label: str, - lexicon_graph: Path, - grammar_graph: Path, -) -> Path: - lg_graph = fst_dir / f"LG.{unique_label}.fst" - - if not lg_graph.exists(): - logger.info(f"Creating {lg_graph}") - - fsttablecompose = kaldi_root / "src/fstbin/fsttablecompose" - fstdeterminizestar = kaldi_root / "src/fstbin/fstdeterminizestar" - fstminimizeencoded = kaldi_root / "src/fstbin/fstminimizeencoded" - fstpushspecial = kaldi_root / "src/fstbin/fstpushspecial" - fstarcsort = kaldi_root / "tools/openfst-1.6.7/bin/fstarcsort" - - try: - with open(lg_graph, "wb") as out_f: - res = subprocess.run( - [fsttablecompose, lexicon_graph, grammar_graph], - capture_output=True, - check=True, - ) - res = subprocess.run( - [ - fstdeterminizestar, - "--use-log=true", - ], - input=res.stdout, - capture_output=True, - ) - res = subprocess.run( - [fstminimizeencoded], - input=res.stdout, - capture_output=True, - check=True, - ) - res = subprocess.run( - [fstpushspecial], - input=res.stdout, - capture_output=True, - check=True, - ) - res = subprocess.run( - [fstarcsort, "--sort_type=ilabel"], - input=res.stdout, - capture_output=True, - check=True, - ) - out_f.write(res.stdout) - except subprocess.CalledProcessError as e: - logger.error(f"cmd: {e.cmd}, err: {e.stderr.decode('utf-8')}") - os.remove(lg_graph) - raise - - return lg_graph - - -def create_H( - kaldi_root: Path, - fst_dir: Path, - disambig_out_units_file: Path, - in_labels: str, - vocab: Dictionary, - blk_sym: str, - silence_symbol: Optional[str], -) -> (Path, Path, Path): - h_graph = ( - fst_dir / f"H.{in_labels}{'_' + silence_symbol if silence_symbol else ''}.fst" - ) - h_out_units_file = fst_dir / f"kaldi_dict.h_out.{in_labels}.txt" - disambig_in_units_file_int = Path(str(h_graph) + "isym_disambig.int") - disambig_out_units_file_int = Path(str(disambig_out_units_file) + ".int") - if ( - not h_graph.exists() - or not h_out_units_file.exists() - or not disambig_in_units_file_int.exists() - ): - logger.info(f"Creating {h_graph}") - eps_sym = "" - - num_disambig = 0 - osymbols = [] - - with open(disambig_out_units_file, "r") as f, open( - disambig_out_units_file_int, "w" - ) as out_f: - for line in f: - symb, id = line.rstrip().split() - if line.startswith("#"): - num_disambig += 1 - print(id, file=out_f) - else: - if len(osymbols) == 0: - assert symb == eps_sym, symb - osymbols.append((symb, id)) - - i_idx = 0 - isymbols = [(eps_sym, 0)] - - imap = {} - - for i, s in enumerate(vocab.symbols): - i_idx += 1 - isymbols.append((s, i_idx)) - imap[s] = i_idx - - fst_str = [] - - node_idx = 0 - root_node = node_idx - - special_symbols = [blk_sym] - if silence_symbol is not None: - special_symbols.append(silence_symbol) - - for ss in special_symbols: - fst_str.append("{} {} {} {}".format(root_node, root_node, ss, eps_sym)) - - for symbol, _ in osymbols: - if symbol == eps_sym or symbol.startswith("#"): - continue - - node_idx += 1 - # 1. from root to emitting state - fst_str.append("{} {} {} {}".format(root_node, node_idx, symbol, symbol)) - # 2. from emitting state back to root - fst_str.append("{} {} {} {}".format(node_idx, root_node, eps_sym, eps_sym)) - # 3. from emitting state to optional blank state - pre_node = node_idx - node_idx += 1 - for ss in special_symbols: - fst_str.append("{} {} {} {}".format(pre_node, node_idx, ss, eps_sym)) - # 4. from blank state back to root - fst_str.append("{} {} {} {}".format(node_idx, root_node, eps_sym, eps_sym)) - - fst_str.append("{}".format(root_node)) - - fst_str = "\n".join(fst_str) - h_str = str(h_graph) - isym_file = h_str + ".isym" - - with open(isym_file, "w") as f: - for sym, id in isymbols: - f.write("{} {}\n".format(sym, id)) - - with open(h_out_units_file, "w") as f: - for sym, id in osymbols: - f.write("{} {}\n".format(sym, id)) - - with open(disambig_in_units_file_int, "w") as f: - disam_sym_id = len(isymbols) - for _ in range(num_disambig): - f.write("{}\n".format(disam_sym_id)) - disam_sym_id += 1 - - fstcompile = kaldi_root / "tools/openfst-1.6.7/bin/fstcompile" - fstaddselfloops = kaldi_root / "src/fstbin/fstaddselfloops" - fstarcsort = kaldi_root / "tools/openfst-1.6.7/bin/fstarcsort" - - try: - with open(h_graph, "wb") as out_f: - res = subprocess.run( - [ - fstcompile, - f"--isymbols={isym_file}", - f"--osymbols={h_out_units_file}", - "--keep_isymbols=false", - "--keep_osymbols=false", - ], - input=str.encode(fst_str), - capture_output=True, - check=True, - ) - res = subprocess.run( - [ - fstaddselfloops, - disambig_in_units_file_int, - disambig_out_units_file_int, - ], - input=res.stdout, - capture_output=True, - check=True, - ) - res = subprocess.run( - [fstarcsort, "--sort_type=olabel"], - input=res.stdout, - capture_output=True, - check=True, - ) - out_f.write(res.stdout) - except subprocess.CalledProcessError as e: - logger.error(f"cmd: {e.cmd}, err: {e.stderr.decode('utf-8')}") - os.remove(h_graph) - raise - return h_graph, h_out_units_file, disambig_in_units_file_int - - -def create_HLGa( - kaldi_root: Path, - fst_dir: Path, - unique_label: str, - h_graph: Path, - lg_graph: Path, - disambig_in_words_file_int: Path, -) -> Path: - hlga_graph = fst_dir / f"HLGa.{unique_label}.fst" - - if not hlga_graph.exists(): - logger.info(f"Creating {hlga_graph}") - - fsttablecompose = kaldi_root / "src/fstbin/fsttablecompose" - fstdeterminizestar = kaldi_root / "src/fstbin/fstdeterminizestar" - fstrmsymbols = kaldi_root / "src/fstbin/fstrmsymbols" - fstrmepslocal = kaldi_root / "src/fstbin/fstrmepslocal" - fstminimizeencoded = kaldi_root / "src/fstbin/fstminimizeencoded" - - try: - with open(hlga_graph, "wb") as out_f: - res = subprocess.run( - [ - fsttablecompose, - h_graph, - lg_graph, - ], - capture_output=True, - check=True, - ) - res = subprocess.run( - [fstdeterminizestar, "--use-log=true"], - input=res.stdout, - capture_output=True, - check=True, - ) - res = subprocess.run( - [fstrmsymbols, disambig_in_words_file_int], - input=res.stdout, - capture_output=True, - check=True, - ) - res = subprocess.run( - [fstrmepslocal], - input=res.stdout, - capture_output=True, - check=True, - ) - res = subprocess.run( - [fstminimizeencoded], - input=res.stdout, - capture_output=True, - check=True, - ) - out_f.write(res.stdout) - except subprocess.CalledProcessError as e: - logger.error(f"cmd: {e.cmd}, err: {e.stderr.decode('utf-8')}") - os.remove(hlga_graph) - raise - - return hlga_graph - - -def create_HLa( - kaldi_root: Path, - fst_dir: Path, - unique_label: str, - h_graph: Path, - l_graph: Path, - disambig_in_words_file_int: Path, -) -> Path: - hla_graph = fst_dir / f"HLa.{unique_label}.fst" - - if not hla_graph.exists(): - logger.info(f"Creating {hla_graph}") - - fsttablecompose = kaldi_root / "src/fstbin/fsttablecompose" - fstdeterminizestar = kaldi_root / "src/fstbin/fstdeterminizestar" - fstrmsymbols = kaldi_root / "src/fstbin/fstrmsymbols" - fstrmepslocal = kaldi_root / "src/fstbin/fstrmepslocal" - fstminimizeencoded = kaldi_root / "src/fstbin/fstminimizeencoded" - - try: - with open(hla_graph, "wb") as out_f: - res = subprocess.run( - [ - fsttablecompose, - h_graph, - l_graph, - ], - capture_output=True, - check=True, - ) - res = subprocess.run( - [fstdeterminizestar, "--use-log=true"], - input=res.stdout, - capture_output=True, - check=True, - ) - res = subprocess.run( - [fstrmsymbols, disambig_in_words_file_int], - input=res.stdout, - capture_output=True, - check=True, - ) - res = subprocess.run( - [fstrmepslocal], - input=res.stdout, - capture_output=True, - check=True, - ) - res = subprocess.run( - [fstminimizeencoded], - input=res.stdout, - capture_output=True, - check=True, - ) - out_f.write(res.stdout) - except subprocess.CalledProcessError as e: - logger.error(f"cmd: {e.cmd}, err: {e.stderr.decode('utf-8')}") - os.remove(hla_graph) - raise - - return hla_graph - - -def create_HLG( - kaldi_root: Path, - fst_dir: Path, - unique_label: str, - hlga_graph: Path, - prefix: str = "HLG", -) -> Path: - hlg_graph = fst_dir / f"{prefix}.{unique_label}.fst" - - if not hlg_graph.exists(): - logger.info(f"Creating {hlg_graph}") - - add_self_loop = script_dir / "add-self-loop-simple" - kaldi_src = kaldi_root / "src" - kaldi_lib = kaldi_src / "lib" - - try: - if not add_self_loop.exists(): - fst_include = kaldi_root / "tools/openfst-1.6.7/include" - add_self_loop_src = script_dir / "add-self-loop-simple.cc" - - subprocess.run( - [ - "c++", - f"-I{kaldi_src}", - f"-I{fst_include}", - f"-L{kaldi_lib}", - add_self_loop_src, - "-lkaldi-base", - "-lkaldi-fstext", - "-o", - add_self_loop, - ], - check=True, - ) - - my_env = os.environ.copy() - my_env["LD_LIBRARY_PATH"] = f"{kaldi_lib}:{my_env['LD_LIBRARY_PATH']}" - - subprocess.run( - [ - add_self_loop, - hlga_graph, - hlg_graph, - ], - check=True, - capture_output=True, - env=my_env, - ) - except subprocess.CalledProcessError as e: - logger.error(f"cmd: {e.cmd}, err: {e.stderr.decode('utf-8')}") - raise - - return hlg_graph - - -def initalize_kaldi(cfg: KaldiInitializerConfig) -> Path: - if cfg.fst_dir is None: - cfg.fst_dir = osp.join(cfg.data_dir, "kaldi") - if cfg.out_labels is None: - cfg.out_labels = cfg.in_labels - - kaldi_root = Path(cfg.kaldi_root) - data_dir = Path(cfg.data_dir) - fst_dir = Path(cfg.fst_dir) - fst_dir.mkdir(parents=True, exist_ok=True) - - arpa_base = osp.splitext(osp.basename(cfg.lm_arpa))[0] - unique_label = f"{cfg.in_labels}.{arpa_base}" - - with open(data_dir / f"dict.{cfg.in_labels}.txt", "r") as f: - vocab = Dictionary.load(f) - - in_units_file = create_units(fst_dir, cfg.in_labels, vocab) - - grammar_graph, out_words_file = create_G( - kaldi_root, fst_dir, Path(cfg.lm_arpa), arpa_base - ) - - disambig_lexicon_file, disambig_L_in_units_file = create_lexicon( - cfg, fst_dir, unique_label, in_units_file, out_words_file - ) - - h_graph, h_out_units_file, disambig_in_units_file_int = create_H( - kaldi_root, - fst_dir, - disambig_L_in_units_file, - cfg.in_labels, - vocab, - cfg.blank_symbol, - cfg.silence_symbol, - ) - lexicon_graph = create_L( - kaldi_root, - fst_dir, - unique_label, - disambig_lexicon_file, - disambig_L_in_units_file, - out_words_file, - ) - lg_graph = create_LG( - kaldi_root, fst_dir, unique_label, lexicon_graph, grammar_graph - ) - hlga_graph = create_HLGa( - kaldi_root, fst_dir, unique_label, h_graph, lg_graph, disambig_in_units_file_int - ) - hlg_graph = create_HLG(kaldi_root, fst_dir, unique_label, hlga_graph) - - # for debugging - # hla_graph = create_HLa(kaldi_root, fst_dir, unique_label, h_graph, lexicon_graph, disambig_in_units_file_int) - # hl_graph = create_HLG(kaldi_root, fst_dir, unique_label, hla_graph, prefix="HL_looped") - # create_HLG(kaldi_root, fst_dir, "phnc", h_graph, prefix="H_looped") - - return hlg_graph - - -@hydra.main(config_path=config_path, config_name="kaldi_initializer") -def cli_main(cfg: KaldiInitializerConfig) -> None: - container = OmegaConf.to_container(cfg, resolve=True, enum_to_str=True) - cfg = OmegaConf.create(container) - OmegaConf.set_struct(cfg, True) - initalize_kaldi(cfg) - - -if __name__ == "__main__": - - logging.root.setLevel(logging.INFO) - logging.basicConfig(level=logging.INFO) - - try: - from hydra._internal.utils import ( # pylint: disable=import-outside-toplevel - get_args, - ) - - cfg_name = get_args().config_name or "kaldi_initializer" - except ImportError: - logger.warning("Failed to get config name from hydra args") - cfg_name = "kaldi_initializer" - - cs = ConfigStore.instance() - cs.store(name=cfg_name, node=KaldiInitializerConfig) - - cli_main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/models/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/models/__init__.py deleted file mode 100644 index 82e71f9af7c8037d3d59c2c247574725c83b8b8d..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/models/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -import importlib -import os - -for file in sorted(os.listdir(os.path.dirname(__file__))): - if file.endswith(".py") and not file.startswith("_"): - model_name = file[: file.find(".py")] - importlib.import_module("examples.speech_recognition.models." + model_name) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/models/vggtransformer.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/models/vggtransformer.py deleted file mode 100644 index bca0ae59a8cbe2b7c337e395021c883a61d101ee..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/models/vggtransformer.py +++ /dev/null @@ -1,1020 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import math -from collections.abc import Iterable - -import torch -import torch.nn as nn -from examples.speech_recognition.data.data_utils import lengths_to_encoder_padding_mask -from fairseq import utils -from fairseq.models import ( - FairseqEncoder, - FairseqEncoderDecoderModel, - FairseqEncoderModel, - FairseqIncrementalDecoder, - register_model, - register_model_architecture, -) -from fairseq.modules import ( - LinearizedConvolution, - TransformerDecoderLayer, - TransformerEncoderLayer, - VGGBlock, -) - - -@register_model("asr_vggtransformer") -class VGGTransformerModel(FairseqEncoderDecoderModel): - """ - Transformers with convolutional context for ASR - https://arxiv.org/abs/1904.11660 - """ - - def __init__(self, encoder, decoder): - super().__init__(encoder, decoder) - - @staticmethod - def add_args(parser): - """Add model-specific arguments to the parser.""" - parser.add_argument( - "--input-feat-per-channel", - type=int, - metavar="N", - help="encoder input dimension per input channel", - ) - parser.add_argument( - "--vggblock-enc-config", - type=str, - metavar="EXPR", - help=""" - an array of tuples each containing the configuration of one vggblock: - [(out_channels, - conv_kernel_size, - pooling_kernel_size, - num_conv_layers, - use_layer_norm), ...]) - """, - ) - parser.add_argument( - "--transformer-enc-config", - type=str, - metavar="EXPR", - help="""" - a tuple containing the configuration of the encoder transformer layers - configurations: - [(input_dim, - num_heads, - ffn_dim, - normalize_before, - dropout, - attention_dropout, - relu_dropout), ...]') - """, - ) - parser.add_argument( - "--enc-output-dim", - type=int, - metavar="N", - help=""" - encoder output dimension, can be None. If specified, projecting the - transformer output to the specified dimension""", - ) - parser.add_argument( - "--in-channels", - type=int, - metavar="N", - help="number of encoder input channels", - ) - parser.add_argument( - "--tgt-embed-dim", - type=int, - metavar="N", - help="embedding dimension of the decoder target tokens", - ) - parser.add_argument( - "--transformer-dec-config", - type=str, - metavar="EXPR", - help=""" - a tuple containing the configuration of the decoder transformer layers - configurations: - [(input_dim, - num_heads, - ffn_dim, - normalize_before, - dropout, - attention_dropout, - relu_dropout), ...] - """, - ) - parser.add_argument( - "--conv-dec-config", - type=str, - metavar="EXPR", - help=""" - an array of tuples for the decoder 1-D convolution config - [(out_channels, conv_kernel_size, use_layer_norm), ...]""", - ) - - @classmethod - def build_encoder(cls, args, task): - return VGGTransformerEncoder( - input_feat_per_channel=args.input_feat_per_channel, - vggblock_config=eval(args.vggblock_enc_config), - transformer_config=eval(args.transformer_enc_config), - encoder_output_dim=args.enc_output_dim, - in_channels=args.in_channels, - ) - - @classmethod - def build_decoder(cls, args, task): - return TransformerDecoder( - dictionary=task.target_dictionary, - embed_dim=args.tgt_embed_dim, - transformer_config=eval(args.transformer_dec_config), - conv_config=eval(args.conv_dec_config), - encoder_output_dim=args.enc_output_dim, - ) - - @classmethod - def build_model(cls, args, task): - """Build a new model instance.""" - # make sure that all args are properly defaulted - # (in case there are any new ones) - base_architecture(args) - - encoder = cls.build_encoder(args, task) - decoder = cls.build_decoder(args, task) - return cls(encoder, decoder) - - def get_normalized_probs(self, net_output, log_probs, sample=None): - # net_output['encoder_out'] is a (B, T, D) tensor - lprobs = super().get_normalized_probs(net_output, log_probs, sample) - lprobs.batch_first = True - return lprobs - - -DEFAULT_ENC_VGGBLOCK_CONFIG = ((32, 3, 2, 2, False),) * 2 -DEFAULT_ENC_TRANSFORMER_CONFIG = ((256, 4, 1024, True, 0.2, 0.2, 0.2),) * 2 -# 256: embedding dimension -# 4: number of heads -# 1024: FFN -# True: apply layerNorm before (dropout + resiaul) instead of after -# 0.2 (dropout): dropout after MultiheadAttention and second FC -# 0.2 (attention_dropout): dropout in MultiheadAttention -# 0.2 (relu_dropout): dropout after ReLu -DEFAULT_DEC_TRANSFORMER_CONFIG = ((256, 2, 1024, True, 0.2, 0.2, 0.2),) * 2 -DEFAULT_DEC_CONV_CONFIG = ((256, 3, True),) * 2 - - -# TODO: repace transformer encoder config from one liner -# to explicit args to get rid of this transformation -def prepare_transformer_encoder_params( - input_dim, - num_heads, - ffn_dim, - normalize_before, - dropout, - attention_dropout, - relu_dropout, -): - args = argparse.Namespace() - args.encoder_embed_dim = input_dim - args.encoder_attention_heads = num_heads - args.attention_dropout = attention_dropout - args.dropout = dropout - args.activation_dropout = relu_dropout - args.encoder_normalize_before = normalize_before - args.encoder_ffn_embed_dim = ffn_dim - return args - - -def prepare_transformer_decoder_params( - input_dim, - num_heads, - ffn_dim, - normalize_before, - dropout, - attention_dropout, - relu_dropout, -): - args = argparse.Namespace() - args.encoder_embed_dim = None - args.decoder_embed_dim = input_dim - args.decoder_attention_heads = num_heads - args.attention_dropout = attention_dropout - args.dropout = dropout - args.activation_dropout = relu_dropout - args.decoder_normalize_before = normalize_before - args.decoder_ffn_embed_dim = ffn_dim - return args - - -class VGGTransformerEncoder(FairseqEncoder): - """VGG + Transformer encoder""" - - def __init__( - self, - input_feat_per_channel, - vggblock_config=DEFAULT_ENC_VGGBLOCK_CONFIG, - transformer_config=DEFAULT_ENC_TRANSFORMER_CONFIG, - encoder_output_dim=512, - in_channels=1, - transformer_context=None, - transformer_sampling=None, - ): - """constructor for VGGTransformerEncoder - - Args: - - input_feat_per_channel: feature dim (not including stacked, - just base feature) - - in_channel: # input channels (e.g., if stack 8 feature vector - together, this is 8) - - vggblock_config: configuration of vggblock, see comments on - DEFAULT_ENC_VGGBLOCK_CONFIG - - transformer_config: configuration of transformer layer, see comments - on DEFAULT_ENC_TRANSFORMER_CONFIG - - encoder_output_dim: final transformer output embedding dimension - - transformer_context: (left, right) if set, self-attention will be focused - on (t-left, t+right) - - transformer_sampling: an iterable of int, must match with - len(transformer_config), transformer_sampling[i] indicates sampling - factor for i-th transformer layer, after multihead att and feedfoward - part - """ - super().__init__(None) - - self.num_vggblocks = 0 - if vggblock_config is not None: - if not isinstance(vggblock_config, Iterable): - raise ValueError("vggblock_config is not iterable") - self.num_vggblocks = len(vggblock_config) - - self.conv_layers = nn.ModuleList() - self.in_channels = in_channels - self.input_dim = input_feat_per_channel - self.pooling_kernel_sizes = [] - - if vggblock_config is not None: - for _, config in enumerate(vggblock_config): - ( - out_channels, - conv_kernel_size, - pooling_kernel_size, - num_conv_layers, - layer_norm, - ) = config - self.conv_layers.append( - VGGBlock( - in_channels, - out_channels, - conv_kernel_size, - pooling_kernel_size, - num_conv_layers, - input_dim=input_feat_per_channel, - layer_norm=layer_norm, - ) - ) - self.pooling_kernel_sizes.append(pooling_kernel_size) - in_channels = out_channels - input_feat_per_channel = self.conv_layers[-1].output_dim - - transformer_input_dim = self.infer_conv_output_dim( - self.in_channels, self.input_dim - ) - # transformer_input_dim is the output dimension of VGG part - - self.validate_transformer_config(transformer_config) - self.transformer_context = self.parse_transformer_context(transformer_context) - self.transformer_sampling = self.parse_transformer_sampling( - transformer_sampling, len(transformer_config) - ) - - self.transformer_layers = nn.ModuleList() - - if transformer_input_dim != transformer_config[0][0]: - self.transformer_layers.append( - Linear(transformer_input_dim, transformer_config[0][0]) - ) - self.transformer_layers.append( - TransformerEncoderLayer( - prepare_transformer_encoder_params(*transformer_config[0]) - ) - ) - - for i in range(1, len(transformer_config)): - if transformer_config[i - 1][0] != transformer_config[i][0]: - self.transformer_layers.append( - Linear(transformer_config[i - 1][0], transformer_config[i][0]) - ) - self.transformer_layers.append( - TransformerEncoderLayer( - prepare_transformer_encoder_params(*transformer_config[i]) - ) - ) - - self.encoder_output_dim = encoder_output_dim - self.transformer_layers.extend( - [ - Linear(transformer_config[-1][0], encoder_output_dim), - LayerNorm(encoder_output_dim), - ] - ) - - def forward(self, src_tokens, src_lengths, **kwargs): - """ - src_tokens: padded tensor (B, T, C * feat) - src_lengths: tensor of original lengths of input utterances (B,) - """ - bsz, max_seq_len, _ = src_tokens.size() - x = src_tokens.view(bsz, max_seq_len, self.in_channels, self.input_dim) - x = x.transpose(1, 2).contiguous() - # (B, C, T, feat) - - for layer_idx in range(len(self.conv_layers)): - x = self.conv_layers[layer_idx](x) - - bsz, _, output_seq_len, _ = x.size() - - # (B, C, T, feat) -> (B, T, C, feat) -> (T, B, C, feat) -> (T, B, C * feat) - x = x.transpose(1, 2).transpose(0, 1) - x = x.contiguous().view(output_seq_len, bsz, -1) - - input_lengths = src_lengths.clone() - for s in self.pooling_kernel_sizes: - input_lengths = (input_lengths.float() / s).ceil().long() - - encoder_padding_mask, _ = lengths_to_encoder_padding_mask( - input_lengths, batch_first=True - ) - if not encoder_padding_mask.any(): - encoder_padding_mask = None - - subsampling_factor = int(max_seq_len * 1.0 / output_seq_len + 0.5) - attn_mask = self.lengths_to_attn_mask(input_lengths, subsampling_factor) - - transformer_layer_idx = 0 - - for layer_idx in range(len(self.transformer_layers)): - - if isinstance(self.transformer_layers[layer_idx], TransformerEncoderLayer): - x = self.transformer_layers[layer_idx]( - x, encoder_padding_mask, attn_mask - ) - - if self.transformer_sampling[transformer_layer_idx] != 1: - sampling_factor = self.transformer_sampling[transformer_layer_idx] - x, encoder_padding_mask, attn_mask = self.slice( - x, encoder_padding_mask, attn_mask, sampling_factor - ) - - transformer_layer_idx += 1 - - else: - x = self.transformer_layers[layer_idx](x) - - # encoder_padding_maks is a (T x B) tensor, its [t, b] elements indicate - # whether encoder_output[t, b] is valid or not (valid=0, invalid=1) - - return { - "encoder_out": x, # (T, B, C) - "encoder_padding_mask": encoder_padding_mask.t() - if encoder_padding_mask is not None - else None, - # (B, T) --> (T, B) - } - - def infer_conv_output_dim(self, in_channels, input_dim): - sample_seq_len = 200 - sample_bsz = 10 - x = torch.randn(sample_bsz, in_channels, sample_seq_len, input_dim) - for i, _ in enumerate(self.conv_layers): - x = self.conv_layers[i](x) - x = x.transpose(1, 2) - mb, seq = x.size()[:2] - return x.contiguous().view(mb, seq, -1).size(-1) - - def validate_transformer_config(self, transformer_config): - for config in transformer_config: - input_dim, num_heads = config[:2] - if input_dim % num_heads != 0: - msg = ( - "ERROR in transformer config {}: ".format(config) - + "input dimension {} ".format(input_dim) - + "not dividable by number of heads {}".format(num_heads) - ) - raise ValueError(msg) - - def parse_transformer_context(self, transformer_context): - """ - transformer_context can be the following: - - None; indicates no context is used, i.e., - transformer can access full context - - a tuple/list of two int; indicates left and right context, - any number <0 indicates infinite context - * e.g., (5, 6) indicates that for query at x_t, transformer can - access [t-5, t+6] (inclusive) - * e.g., (-1, 6) indicates that for query at x_t, transformer can - access [0, t+6] (inclusive) - """ - if transformer_context is None: - return None - - if not isinstance(transformer_context, Iterable): - raise ValueError("transformer context must be Iterable if it is not None") - - if len(transformer_context) != 2: - raise ValueError("transformer context must have length 2") - - left_context = transformer_context[0] - if left_context < 0: - left_context = None - - right_context = transformer_context[1] - if right_context < 0: - right_context = None - - if left_context is None and right_context is None: - return None - - return (left_context, right_context) - - def parse_transformer_sampling(self, transformer_sampling, num_layers): - """ - parsing transformer sampling configuration - - Args: - - transformer_sampling, accepted input: - * None, indicating no sampling - * an Iterable with int (>0) as element - - num_layers, expected number of transformer layers, must match with - the length of transformer_sampling if it is not None - - Returns: - - A tuple with length num_layers - """ - if transformer_sampling is None: - return (1,) * num_layers - - if not isinstance(transformer_sampling, Iterable): - raise ValueError( - "transformer_sampling must be an iterable if it is not None" - ) - - if len(transformer_sampling) != num_layers: - raise ValueError( - "transformer_sampling {} does not match with the number " - "of layers {}".format(transformer_sampling, num_layers) - ) - - for layer, value in enumerate(transformer_sampling): - if not isinstance(value, int): - raise ValueError("Invalid value in transformer_sampling: ") - if value < 1: - raise ValueError( - "{} layer's subsampling is {}.".format(layer, value) - + " This is not allowed! " - ) - return transformer_sampling - - def slice(self, embedding, padding_mask, attn_mask, sampling_factor): - """ - embedding is a (T, B, D) tensor - padding_mask is a (B, T) tensor or None - attn_mask is a (T, T) tensor or None - """ - embedding = embedding[::sampling_factor, :, :] - if padding_mask is not None: - padding_mask = padding_mask[:, ::sampling_factor] - if attn_mask is not None: - attn_mask = attn_mask[::sampling_factor, ::sampling_factor] - - return embedding, padding_mask, attn_mask - - def lengths_to_attn_mask(self, input_lengths, subsampling_factor=1): - """ - create attention mask according to sequence lengths and transformer - context - - Args: - - input_lengths: (B, )-shape Int/Long tensor; input_lengths[b] is - the length of b-th sequence - - subsampling_factor: int - * Note that the left_context and right_context is specified in - the input frame-level while input to transformer may already - go through subsampling (e.g., the use of striding in vggblock) - we use subsampling_factor to scale the left/right context - - Return: - - a (T, T) binary tensor or None, where T is max(input_lengths) - * if self.transformer_context is None, None - * if left_context is None, - * attn_mask[t, t + right_context + 1:] = 1 - * others = 0 - * if right_context is None, - * attn_mask[t, 0:t - left_context] = 1 - * others = 0 - * elsif - * attn_mask[t, t - left_context: t + right_context + 1] = 0 - * others = 1 - """ - if self.transformer_context is None: - return None - - maxT = torch.max(input_lengths).item() - attn_mask = torch.zeros(maxT, maxT) - - left_context = self.transformer_context[0] - right_context = self.transformer_context[1] - if left_context is not None: - left_context = math.ceil(self.transformer_context[0] / subsampling_factor) - if right_context is not None: - right_context = math.ceil(self.transformer_context[1] / subsampling_factor) - - for t in range(maxT): - if left_context is not None: - st = 0 - en = max(st, t - left_context) - attn_mask[t, st:en] = 1 - if right_context is not None: - st = t + right_context + 1 - st = min(st, maxT - 1) - attn_mask[t, st:] = 1 - - return attn_mask.to(input_lengths.device) - - def reorder_encoder_out(self, encoder_out, new_order): - encoder_out["encoder_out"] = encoder_out["encoder_out"].index_select( - 1, new_order - ) - if encoder_out["encoder_padding_mask"] is not None: - encoder_out["encoder_padding_mask"] = encoder_out[ - "encoder_padding_mask" - ].index_select(1, new_order) - return encoder_out - - -class TransformerDecoder(FairseqIncrementalDecoder): - """ - Transformer decoder consisting of *args.decoder_layers* layers. Each layer - is a :class:`TransformerDecoderLayer`. - Args: - args (argparse.Namespace): parsed command-line arguments - dictionary (~fairseq.data.Dictionary): decoding dictionary - embed_tokens (torch.nn.Embedding): output embedding - no_encoder_attn (bool, optional): whether to attend to encoder outputs. - Default: ``False`` - left_pad (bool, optional): whether the input is left-padded. Default: - ``False`` - """ - - def __init__( - self, - dictionary, - embed_dim=512, - transformer_config=DEFAULT_ENC_TRANSFORMER_CONFIG, - conv_config=DEFAULT_DEC_CONV_CONFIG, - encoder_output_dim=512, - ): - - super().__init__(dictionary) - vocab_size = len(dictionary) - self.padding_idx = dictionary.pad() - self.embed_tokens = Embedding(vocab_size, embed_dim, self.padding_idx) - - self.conv_layers = nn.ModuleList() - for i in range(len(conv_config)): - out_channels, kernel_size, layer_norm = conv_config[i] - if i == 0: - conv_layer = LinearizedConv1d( - embed_dim, out_channels, kernel_size, padding=kernel_size - 1 - ) - else: - conv_layer = LinearizedConv1d( - conv_config[i - 1][0], - out_channels, - kernel_size, - padding=kernel_size - 1, - ) - self.conv_layers.append(conv_layer) - if layer_norm: - self.conv_layers.append(nn.LayerNorm(out_channels)) - self.conv_layers.append(nn.ReLU()) - - self.layers = nn.ModuleList() - if conv_config[-1][0] != transformer_config[0][0]: - self.layers.append(Linear(conv_config[-1][0], transformer_config[0][0])) - self.layers.append( - TransformerDecoderLayer( - prepare_transformer_decoder_params(*transformer_config[0]) - ) - ) - - for i in range(1, len(transformer_config)): - if transformer_config[i - 1][0] != transformer_config[i][0]: - self.layers.append( - Linear(transformer_config[i - 1][0], transformer_config[i][0]) - ) - self.layers.append( - TransformerDecoderLayer( - prepare_transformer_decoder_params(*transformer_config[i]) - ) - ) - self.fc_out = Linear(transformer_config[-1][0], vocab_size) - - def forward(self, prev_output_tokens, encoder_out=None, incremental_state=None): - """ - Args: - prev_output_tokens (LongTensor): previous decoder outputs of shape - `(batch, tgt_len)`, for input feeding/teacher forcing - encoder_out (Tensor, optional): output from the encoder, used for - encoder-side attention - incremental_state (dict): dictionary used for storing state during - :ref:`Incremental decoding` - Returns: - tuple: - - the last decoder layer's output of shape `(batch, tgt_len, - vocab)` - - the last decoder layer's attention weights of shape `(batch, - tgt_len, src_len)` - """ - target_padding_mask = ( - (prev_output_tokens == self.padding_idx).to(prev_output_tokens.device) - if incremental_state is None - else None - ) - - if incremental_state is not None: - prev_output_tokens = prev_output_tokens[:, -1:] - - # embed tokens - x = self.embed_tokens(prev_output_tokens) - - # B x T x C -> T x B x C - x = self._transpose_if_training(x, incremental_state) - - for layer in self.conv_layers: - if isinstance(layer, LinearizedConvolution): - x = layer(x, incremental_state) - else: - x = layer(x) - - # B x T x C -> T x B x C - x = self._transpose_if_inference(x, incremental_state) - - # decoder layers - for layer in self.layers: - if isinstance(layer, TransformerDecoderLayer): - x, *_ = layer( - x, - (encoder_out["encoder_out"] if encoder_out is not None else None), - ( - encoder_out["encoder_padding_mask"].t() - if encoder_out["encoder_padding_mask"] is not None - else None - ), - incremental_state, - self_attn_mask=( - self.buffered_future_mask(x) - if incremental_state is None - else None - ), - self_attn_padding_mask=( - target_padding_mask if incremental_state is None else None - ), - ) - else: - x = layer(x) - - # T x B x C -> B x T x C - x = x.transpose(0, 1) - - x = self.fc_out(x) - - return x, None - - def buffered_future_mask(self, tensor): - dim = tensor.size(0) - if ( - not hasattr(self, "_future_mask") - or self._future_mask is None - or self._future_mask.device != tensor.device - ): - self._future_mask = torch.triu( - utils.fill_with_neg_inf(tensor.new(dim, dim)), 1 - ) - if self._future_mask.size(0) < dim: - self._future_mask = torch.triu( - utils.fill_with_neg_inf(self._future_mask.resize_(dim, dim)), 1 - ) - return self._future_mask[:dim, :dim] - - def _transpose_if_training(self, x, incremental_state): - if incremental_state is None: - x = x.transpose(0, 1) - return x - - def _transpose_if_inference(self, x, incremental_state): - if incremental_state: - x = x.transpose(0, 1) - return x - - -@register_model("asr_vggtransformer_encoder") -class VGGTransformerEncoderModel(FairseqEncoderModel): - def __init__(self, encoder): - super().__init__(encoder) - - @staticmethod - def add_args(parser): - """Add model-specific arguments to the parser.""" - parser.add_argument( - "--input-feat-per-channel", - type=int, - metavar="N", - help="encoder input dimension per input channel", - ) - parser.add_argument( - "--vggblock-enc-config", - type=str, - metavar="EXPR", - help=""" - an array of tuples each containing the configuration of one vggblock - [(out_channels, conv_kernel_size, pooling_kernel_size,num_conv_layers), ...] - """, - ) - parser.add_argument( - "--transformer-enc-config", - type=str, - metavar="EXPR", - help=""" - a tuple containing the configuration of the Transformer layers - configurations: - [(input_dim, - num_heads, - ffn_dim, - normalize_before, - dropout, - attention_dropout, - relu_dropout), ]""", - ) - parser.add_argument( - "--enc-output-dim", - type=int, - metavar="N", - help="encoder output dimension, projecting the LSTM output", - ) - parser.add_argument( - "--in-channels", - type=int, - metavar="N", - help="number of encoder input channels", - ) - parser.add_argument( - "--transformer-context", - type=str, - metavar="EXPR", - help=""" - either None or a tuple of two ints, indicating left/right context a - transformer can have access to""", - ) - parser.add_argument( - "--transformer-sampling", - type=str, - metavar="EXPR", - help=""" - either None or a tuple of ints, indicating sampling factor in each layer""", - ) - - @classmethod - def build_model(cls, args, task): - """Build a new model instance.""" - base_architecture_enconly(args) - encoder = VGGTransformerEncoderOnly( - vocab_size=len(task.target_dictionary), - input_feat_per_channel=args.input_feat_per_channel, - vggblock_config=eval(args.vggblock_enc_config), - transformer_config=eval(args.transformer_enc_config), - encoder_output_dim=args.enc_output_dim, - in_channels=args.in_channels, - transformer_context=eval(args.transformer_context), - transformer_sampling=eval(args.transformer_sampling), - ) - return cls(encoder) - - def get_normalized_probs(self, net_output, log_probs, sample=None): - # net_output['encoder_out'] is a (T, B, D) tensor - lprobs = super().get_normalized_probs(net_output, log_probs, sample) - # lprobs is a (T, B, D) tensor - # we need to transoose to get (B, T, D) tensor - lprobs = lprobs.transpose(0, 1).contiguous() - lprobs.batch_first = True - return lprobs - - -class VGGTransformerEncoderOnly(VGGTransformerEncoder): - def __init__( - self, - vocab_size, - input_feat_per_channel, - vggblock_config=DEFAULT_ENC_VGGBLOCK_CONFIG, - transformer_config=DEFAULT_ENC_TRANSFORMER_CONFIG, - encoder_output_dim=512, - in_channels=1, - transformer_context=None, - transformer_sampling=None, - ): - super().__init__( - input_feat_per_channel=input_feat_per_channel, - vggblock_config=vggblock_config, - transformer_config=transformer_config, - encoder_output_dim=encoder_output_dim, - in_channels=in_channels, - transformer_context=transformer_context, - transformer_sampling=transformer_sampling, - ) - self.fc_out = Linear(self.encoder_output_dim, vocab_size) - - def forward(self, src_tokens, src_lengths, **kwargs): - """ - src_tokens: padded tensor (B, T, C * feat) - src_lengths: tensor of original lengths of input utterances (B,) - """ - - enc_out = super().forward(src_tokens, src_lengths) - x = self.fc_out(enc_out["encoder_out"]) - # x = F.log_softmax(x, dim=-1) - # Note: no need this line, because model.get_normalized_prob will call - # log_softmax - return { - "encoder_out": x, # (T, B, C) - "encoder_padding_mask": enc_out["encoder_padding_mask"], # (T, B) - } - - def max_positions(self): - """Maximum input length supported by the encoder.""" - return (1e6, 1e6) # an arbitrary large number - - -def Embedding(num_embeddings, embedding_dim, padding_idx): - m = nn.Embedding(num_embeddings, embedding_dim, padding_idx=padding_idx) - # nn.init.uniform_(m.weight, -0.1, 0.1) - # nn.init.constant_(m.weight[padding_idx], 0) - return m - - -def Linear(in_features, out_features, bias=True, dropout=0): - """Linear layer (input: N x T x C)""" - m = nn.Linear(in_features, out_features, bias=bias) - # m.weight.data.uniform_(-0.1, 0.1) - # if bias: - # m.bias.data.uniform_(-0.1, 0.1) - return m - - -def LinearizedConv1d(in_channels, out_channels, kernel_size, dropout=0, **kwargs): - """Weight-normalized Conv1d layer optimized for decoding""" - m = LinearizedConvolution(in_channels, out_channels, kernel_size, **kwargs) - std = math.sqrt((4 * (1.0 - dropout)) / (m.kernel_size[0] * in_channels)) - nn.init.normal_(m.weight, mean=0, std=std) - nn.init.constant_(m.bias, 0) - return nn.utils.weight_norm(m, dim=2) - - -def LayerNorm(embedding_dim): - m = nn.LayerNorm(embedding_dim) - return m - - -# seq2seq models -def base_architecture(args): - args.input_feat_per_channel = getattr(args, "input_feat_per_channel", 40) - args.vggblock_enc_config = getattr( - args, "vggblock_enc_config", DEFAULT_ENC_VGGBLOCK_CONFIG - ) - args.transformer_enc_config = getattr( - args, "transformer_enc_config", DEFAULT_ENC_TRANSFORMER_CONFIG - ) - args.enc_output_dim = getattr(args, "enc_output_dim", 512) - args.in_channels = getattr(args, "in_channels", 1) - args.tgt_embed_dim = getattr(args, "tgt_embed_dim", 128) - args.transformer_dec_config = getattr( - args, "transformer_dec_config", DEFAULT_ENC_TRANSFORMER_CONFIG - ) - args.conv_dec_config = getattr(args, "conv_dec_config", DEFAULT_DEC_CONV_CONFIG) - args.transformer_context = getattr(args, "transformer_context", "None") - - -@register_model_architecture("asr_vggtransformer", "vggtransformer_1") -def vggtransformer_1(args): - args.input_feat_per_channel = getattr(args, "input_feat_per_channel", 80) - args.vggblock_enc_config = getattr( - args, "vggblock_enc_config", "[(64, 3, 2, 2, True), (128, 3, 2, 2, True)]" - ) - args.transformer_enc_config = getattr( - args, - "transformer_enc_config", - "((1024, 16, 4096, True, 0.15, 0.15, 0.15),) * 14", - ) - args.enc_output_dim = getattr(args, "enc_output_dim", 1024) - args.tgt_embed_dim = getattr(args, "tgt_embed_dim", 128) - args.conv_dec_config = getattr(args, "conv_dec_config", "((256, 3, True),) * 4") - args.transformer_dec_config = getattr( - args, - "transformer_dec_config", - "((1024, 16, 4096, True, 0.15, 0.15, 0.15),) * 4", - ) - - -@register_model_architecture("asr_vggtransformer", "vggtransformer_2") -def vggtransformer_2(args): - args.input_feat_per_channel = getattr(args, "input_feat_per_channel", 80) - args.vggblock_enc_config = getattr( - args, "vggblock_enc_config", "[(64, 3, 2, 2, True), (128, 3, 2, 2, True)]" - ) - args.transformer_enc_config = getattr( - args, - "transformer_enc_config", - "((1024, 16, 4096, True, 0.15, 0.15, 0.15),) * 16", - ) - args.enc_output_dim = getattr(args, "enc_output_dim", 1024) - args.tgt_embed_dim = getattr(args, "tgt_embed_dim", 512) - args.conv_dec_config = getattr(args, "conv_dec_config", "((256, 3, True),) * 4") - args.transformer_dec_config = getattr( - args, - "transformer_dec_config", - "((1024, 16, 4096, True, 0.15, 0.15, 0.15),) * 6", - ) - - -@register_model_architecture("asr_vggtransformer", "vggtransformer_base") -def vggtransformer_base(args): - args.input_feat_per_channel = getattr(args, "input_feat_per_channel", 80) - args.vggblock_enc_config = getattr( - args, "vggblock_enc_config", "[(64, 3, 2, 2, True), (128, 3, 2, 2, True)]" - ) - args.transformer_enc_config = getattr( - args, "transformer_enc_config", "((512, 8, 2048, True, 0.15, 0.15, 0.15),) * 12" - ) - - args.enc_output_dim = getattr(args, "enc_output_dim", 512) - args.tgt_embed_dim = getattr(args, "tgt_embed_dim", 512) - args.conv_dec_config = getattr(args, "conv_dec_config", "((256, 3, True),) * 4") - args.transformer_dec_config = getattr( - args, "transformer_dec_config", "((512, 8, 2048, True, 0.15, 0.15, 0.15),) * 6" - ) - # Size estimations: - # Encoder: - # - vggblock param: 64*1*3*3 + 64*64*3*3 + 128*64*3*3 + 128*128*3 = 258K - # Transformer: - # - input dimension adapter: 2560 x 512 -> 1.31M - # - transformer_layers (x12) --> 37.74M - # * MultiheadAttention: 512*512*3 (in_proj) + 512*512 (out_proj) = 1.048M - # * FFN weight: 512*2048*2 = 2.097M - # - output dimension adapter: 512 x 512 -> 0.26 M - # Decoder: - # - LinearizedConv1d: 512 * 256 * 3 + 256 * 256 * 3 * 3 - # - transformer_layer: (x6) --> 25.16M - # * MultiheadAttention (self-attention): 512*512*3 + 512*512 = 1.048M - # * MultiheadAttention (encoder-attention): 512*512*3 + 512*512 = 1.048M - # * FFN: 512*2048*2 = 2.097M - # Final FC: - # - FC: 512*5000 = 256K (assuming vocab size 5K) - # In total: - # ~65 M - - -# CTC models -def base_architecture_enconly(args): - args.input_feat_per_channel = getattr(args, "input_feat_per_channel", 40) - args.vggblock_enc_config = getattr( - args, "vggblock_enc_config", "[(32, 3, 2, 2, True)] * 2" - ) - args.transformer_enc_config = getattr( - args, "transformer_enc_config", "((256, 4, 1024, True, 0.2, 0.2, 0.2),) * 2" - ) - args.enc_output_dim = getattr(args, "enc_output_dim", 512) - args.in_channels = getattr(args, "in_channels", 1) - args.transformer_context = getattr(args, "transformer_context", "None") - args.transformer_sampling = getattr(args, "transformer_sampling", "None") - - -@register_model_architecture("asr_vggtransformer_encoder", "vggtransformer_enc_1") -def vggtransformer_enc_1(args): - # vggtransformer_1 is the same as vggtransformer_enc_big, except the number - # of layers is increased to 16 - # keep it here for backward compatiablity purpose - args.input_feat_per_channel = getattr(args, "input_feat_per_channel", 80) - args.vggblock_enc_config = getattr( - args, "vggblock_enc_config", "[(64, 3, 2, 2, True), (128, 3, 2, 2, True)]" - ) - args.transformer_enc_config = getattr( - args, - "transformer_enc_config", - "((1024, 16, 4096, True, 0.15, 0.15, 0.15),) * 16", - ) - args.enc_output_dim = getattr(args, "enc_output_dim", 1024) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/models/w2l_conv_glu_enc.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/models/w2l_conv_glu_enc.py deleted file mode 100644 index c213a2f386c2c7355ef0fb80ccc758dddd6f94b9..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/models/w2l_conv_glu_enc.py +++ /dev/null @@ -1,176 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import math - -import torch -import torch.nn as nn -import torch.nn.functional as F -from fairseq.models import ( - FairseqEncoder, - FairseqEncoderModel, - register_model, - register_model_architecture, -) -from fairseq.modules.fairseq_dropout import FairseqDropout - -default_conv_enc_config = """[ - (400, 13, 170, 0.2), - (440, 14, 0, 0.214), - (484, 15, 0, 0.22898), - (532, 16, 0, 0.2450086), - (584, 17, 0, 0.262159202), - (642, 18, 0, 0.28051034614), - (706, 19, 0, 0.30014607037), - (776, 20, 0, 0.321156295296), - (852, 21, 0, 0.343637235966), - (936, 22, 0, 0.367691842484), - (1028, 23, 0, 0.393430271458), - (1130, 24, 0, 0.42097039046), - (1242, 25, 0, 0.450438317792), - (1366, 26, 0, 0.481969000038), - (1502, 27, 0, 0.51570683004), - (1652, 28, 0, 0.551806308143), - (1816, 29, 0, 0.590432749713), -]""" - - -@register_model("asr_w2l_conv_glu_encoder") -class W2lConvGluEncoderModel(FairseqEncoderModel): - def __init__(self, encoder): - super().__init__(encoder) - - @staticmethod - def add_args(parser): - """Add model-specific arguments to the parser.""" - parser.add_argument( - "--input-feat-per-channel", - type=int, - metavar="N", - help="encoder input dimension per input channel", - ) - parser.add_argument( - "--in-channels", - type=int, - metavar="N", - help="number of encoder input channels", - ) - parser.add_argument( - "--conv-enc-config", - type=str, - metavar="EXPR", - help=""" - an array of tuples each containing the configuration of one conv layer - [(out_channels, kernel_size, padding, dropout), ...] - """, - ) - - @classmethod - def build_model(cls, args, task): - """Build a new model instance.""" - conv_enc_config = getattr(args, "conv_enc_config", default_conv_enc_config) - encoder = W2lConvGluEncoder( - vocab_size=len(task.target_dictionary), - input_feat_per_channel=args.input_feat_per_channel, - in_channels=args.in_channels, - conv_enc_config=eval(conv_enc_config), - ) - return cls(encoder) - - def get_normalized_probs(self, net_output, log_probs, sample=None): - lprobs = super().get_normalized_probs(net_output, log_probs, sample) - lprobs.batch_first = False - return lprobs - - -class W2lConvGluEncoder(FairseqEncoder): - def __init__( - self, vocab_size, input_feat_per_channel, in_channels, conv_enc_config - ): - super().__init__(None) - - self.input_dim = input_feat_per_channel - if in_channels != 1: - raise ValueError("only 1 input channel is currently supported") - - self.conv_layers = nn.ModuleList() - self.linear_layers = nn.ModuleList() - self.dropouts = [] - cur_channels = input_feat_per_channel - - for out_channels, kernel_size, padding, dropout in conv_enc_config: - layer = nn.Conv1d(cur_channels, out_channels, kernel_size, padding=padding) - layer.weight.data.mul_(math.sqrt(3)) # match wav2letter init - self.conv_layers.append(nn.utils.weight_norm(layer)) - self.dropouts.append( - FairseqDropout(dropout, module_name=self.__class__.__name__) - ) - if out_channels % 2 != 0: - raise ValueError("odd # of out_channels is incompatible with GLU") - cur_channels = out_channels // 2 # halved by GLU - - for out_channels in [2 * cur_channels, vocab_size]: - layer = nn.Linear(cur_channels, out_channels) - layer.weight.data.mul_(math.sqrt(3)) - self.linear_layers.append(nn.utils.weight_norm(layer)) - cur_channels = out_channels // 2 - - def forward(self, src_tokens, src_lengths, **kwargs): - - """ - src_tokens: padded tensor (B, T, C * feat) - src_lengths: tensor of original lengths of input utterances (B,) - """ - B, T, _ = src_tokens.size() - x = src_tokens.transpose(1, 2).contiguous() # (B, feat, T) assuming C == 1 - - for layer_idx in range(len(self.conv_layers)): - x = self.conv_layers[layer_idx](x) - x = F.glu(x, dim=1) - x = self.dropouts[layer_idx](x) - - x = x.transpose(1, 2).contiguous() # (B, T, 908) - x = self.linear_layers[0](x) - x = F.glu(x, dim=2) - x = self.dropouts[-1](x) - x = self.linear_layers[1](x) - - assert x.size(0) == B - assert x.size(1) == T - - encoder_out = x.transpose(0, 1) # (T, B, vocab_size) - - # need to debug this -- find a simpler/elegant way in pytorch APIs - encoder_padding_mask = ( - torch.arange(T).view(1, T).expand(B, -1).to(x.device) - >= src_lengths.view(B, 1).expand(-1, T) - ).t() # (B x T) -> (T x B) - - return { - "encoder_out": encoder_out, # (T, B, vocab_size) - "encoder_padding_mask": encoder_padding_mask, # (T, B) - } - - def reorder_encoder_out(self, encoder_out, new_order): - encoder_out["encoder_out"] = encoder_out["encoder_out"].index_select( - 1, new_order - ) - encoder_out["encoder_padding_mask"] = encoder_out[ - "encoder_padding_mask" - ].index_select(1, new_order) - return encoder_out - - def max_positions(self): - """Maximum input length supported by the encoder.""" - return (1e6, 1e6) # an arbitrary large number - - -@register_model_architecture("asr_w2l_conv_glu_encoder", "w2l_conv_glu_enc") -def w2l_conv_glu_enc(args): - args.input_feat_per_channel = getattr(args, "input_feat_per_channel", 80) - args.in_channels = getattr(args, "in_channels", 1) - args.conv_enc_config = getattr(args, "conv_enc_config", default_conv_enc_config) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/new/README.md b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/new/README.md deleted file mode 100644 index 5fa0e97245d3ba6db69d11222261b0644960183d..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/new/README.md +++ /dev/null @@ -1,43 +0,0 @@ -# Flashlight Decoder - -This script runs decoding for pre-trained speech recognition models. - -## Usage - -Assuming a few variables: - -```bash -checkpoint= -data= -lm_model= -lexicon= -``` - -Example usage for decoding a fine-tuned Wav2Vec model: - -```bash -python $FAIRSEQ_ROOT/examples/speech_recognition/new/infer.py --multirun \ - task=audio_pretraining \ - task.data=$data \ - task.labels=ltr \ - common_eval.path=$checkpoint \ - decoding.type=kenlm \ - decoding.lexicon=$lexicon \ - decoding.lmpath=$lm_model \ - dataset.gen_subset=dev_clean,dev_other,test_clean,test_other -``` - -Example usage for using Ax to sweep WER parameters (requires `pip install hydra-ax-sweeper`): - -```bash -python $FAIRSEQ_ROOT/examples/speech_recognition/new/infer.py --multirun \ - hydra/sweeper=ax \ - task=audio_pretraining \ - task.data=$data \ - task.labels=ltr \ - common_eval.path=$checkpoint \ - decoding.type=kenlm \ - decoding.lexicon=$lexicon \ - decoding.lmpath=$lm_model \ - dataset.gen_subset=dev_other -``` diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/new/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/new/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/new/conf/hydra/sweeper/ax.yaml b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/new/conf/hydra/sweeper/ax.yaml deleted file mode 100644 index 38e9c221db234877056f59e5d2881199a3d601b4..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/new/conf/hydra/sweeper/ax.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# @package hydra.sweeper -_target_: hydra_plugins.hydra_ax_sweeper.ax_sweeper.AxSweeper -max_batch_size: null -ax_config: - max_trials: 128 - early_stop: - minimize: true - max_epochs_without_improvement: 10 - epsilon: 0.025 - experiment: - name: ${dataset.gen_subset} - objective_name: wer - minimize: true - parameter_constraints: null - outcome_constraints: null - status_quo: null - client: - verbose_logging: false - random_seed: null - params: - decoding.lmweight: - type: range - bounds: [0.0, 5.0] - decoding.wordscore: - type: range - bounds: [-5.0, 5.0] - decoding.silweight: - type: range - bounds: [ -8.0, 0.0 ] diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/new/conf/infer.yaml b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/new/conf/infer.yaml deleted file mode 100644 index 21dd19fadd7cc828455241b89e9c60852b425799..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/new/conf/infer.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# @package _group_ - -defaults: - - task: null - - model: null - -hydra: - run: - dir: ${common_eval.results_path}/${dataset.gen_subset} - sweep: - dir: /checkpoint/${env:USER}/${env:PREFIX}/${common_eval.results_path} - subdir: ${dataset.gen_subset} -common_eval: - results_path: null - path: null - post_process: letter - quiet: true -dataset: - max_tokens: 3000000 - gen_subset: test -distributed_training: - distributed_world_size: 1 -decoding: - beam: 5 - type: viterbi diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/new/decoders/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/new/decoders/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/new/decoders/base_decoder.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/new/decoders/base_decoder.py deleted file mode 100644 index a097969b3c0650cf8ea2ab5f8e96bbc68ea9b97f..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/new/decoders/base_decoder.py +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import itertools as it -from typing import Any, Dict, List - -import torch -from fairseq.data.dictionary import Dictionary -from fairseq.models.fairseq_model import FairseqModel - - -class BaseDecoder: - def __init__(self, tgt_dict: Dictionary) -> None: - self.tgt_dict = tgt_dict - self.vocab_size = len(tgt_dict) - - self.blank = ( - tgt_dict.index("") - if "" in tgt_dict.indices - else tgt_dict.bos() - ) - if "" in tgt_dict.indices: - self.silence = tgt_dict.index("") - elif "|" in tgt_dict.indices: - self.silence = tgt_dict.index("|") - else: - self.silence = tgt_dict.eos() - - def generate( - self, models: List[FairseqModel], sample: Dict[str, Any], **unused - ) -> List[List[Dict[str, torch.LongTensor]]]: - encoder_input = { - k: v for k, v in sample["net_input"].items() if k != "prev_output_tokens" - } - emissions = self.get_emissions(models, encoder_input) - return self.decode(emissions) - - def get_emissions( - self, - models: List[FairseqModel], - encoder_input: Dict[str, Any], - ) -> torch.FloatTensor: - model = models[0] - encoder_out = model(**encoder_input) - if hasattr(model, "get_logits"): - emissions = model.get_logits(encoder_out) - else: - emissions = model.get_normalized_probs(encoder_out, log_probs=True) - return emissions.transpose(0, 1).float().cpu().contiguous() - - def get_tokens(self, idxs: torch.IntTensor) -> torch.LongTensor: - idxs = (g[0] for g in it.groupby(idxs)) - idxs = filter(lambda x: x != self.blank, idxs) - return torch.LongTensor(list(idxs)) - - def decode( - self, - emissions: torch.FloatTensor, - ) -> List[List[Dict[str, torch.LongTensor]]]: - raise NotImplementedError diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/new/decoders/decoder.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/new/decoders/decoder.py deleted file mode 100644 index 43bbe3ececff75cbd203ea10505d041faa5a8041..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/new/decoders/decoder.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from typing import Union - -from fairseq.data.dictionary import Dictionary - -from .base_decoder import BaseDecoder -from .decoder_config import DecoderConfig, FlashlightDecoderConfig - - -def Decoder( - cfg: Union[DecoderConfig, FlashlightDecoderConfig], tgt_dict: Dictionary -) -> BaseDecoder: - - if cfg.type == "viterbi": - from .viterbi_decoder import ViterbiDecoder - - return ViterbiDecoder(tgt_dict) - if cfg.type == "kenlm": - from .flashlight_decoder import KenLMDecoder - - return KenLMDecoder(cfg, tgt_dict) - if cfg.type == "fairseqlm": - from .flashlight_decoder import FairseqLMDecoder - - return FairseqLMDecoder(cfg, tgt_dict) - raise NotImplementedError(f"Invalid decoder name: {cfg.name}") diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/new/decoders/decoder_config.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/new/decoders/decoder_config.py deleted file mode 100644 index e86830771b575d0fd4eac494126839631a5dc50e..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/new/decoders/decoder_config.py +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import math -from dataclasses import dataclass, field -from typing import Optional - -from fairseq.dataclass.configs import FairseqDataclass -from fairseq.dataclass.constants import ChoiceEnum -from omegaconf import MISSING - -DECODER_CHOICES = ChoiceEnum(["viterbi", "kenlm", "fairseqlm"]) - - -@dataclass -class DecoderConfig(FairseqDataclass): - type: DECODER_CHOICES = field( - default="viterbi", - metadata={"help": "The type of decoder to use"}, - ) - - -@dataclass -class FlashlightDecoderConfig(FairseqDataclass): - nbest: int = field( - default=1, - metadata={"help": "Number of decodings to return"}, - ) - unitlm: bool = field( - default=False, - metadata={"help": "If set, use unit language model"}, - ) - lmpath: str = field( - default=MISSING, - metadata={"help": "Language model for KenLM decoder"}, - ) - lexicon: Optional[str] = field( - default=None, - metadata={"help": "Lexicon for Flashlight decoder"}, - ) - beam: int = field( - default=50, - metadata={"help": "Number of beams to use for decoding"}, - ) - beamthreshold: float = field( - default=50.0, - metadata={"help": "Threshold for beam search decoding"}, - ) - beamsizetoken: Optional[int] = field( - default=None, metadata={"help": "Beam size to use"} - ) - wordscore: float = field( - default=-1, - metadata={"help": "Word score for KenLM decoder"}, - ) - unkweight: float = field( - default=-math.inf, - metadata={"help": "Unknown weight for KenLM decoder"}, - ) - silweight: float = field( - default=0, - metadata={"help": "Silence weight for KenLM decoder"}, - ) - lmweight: float = field( - default=2, - metadata={"help": "Weight for LM while interpolating score"}, - ) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/new/decoders/flashlight_decoder.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/new/decoders/flashlight_decoder.py deleted file mode 100644 index b995ae8e0438259aa9dcd0a8ea6b908bfd240d43..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/new/decoders/flashlight_decoder.py +++ /dev/null @@ -1,429 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import gc -import os.path as osp -import warnings -from collections import deque, namedtuple -from typing import Any, Dict, List, Tuple - -import numpy as np -import torch -from fairseq import tasks -from fairseq.data.dictionary import Dictionary -from fairseq.dataclass.utils import convert_namespace_to_omegaconf -from fairseq.models.fairseq_model import FairseqModel -from fairseq.utils import apply_to_sample -from omegaconf import OmegaConf, open_dict - -from .base_decoder import BaseDecoder -from .decoder_config import FlashlightDecoderConfig - -try: - from flashlight.lib.text.decoder import ( - LM, - CriterionType, - DecodeResult, - KenLM, - LexiconDecoder, - LexiconDecoderOptions, - LexiconFreeDecoder, - LexiconFreeDecoderOptions, - LMState, - SmearingMode, - Trie, - ) - from flashlight.lib.text.dictionary import create_word_dict, load_words -except ImportError: - warnings.warn( - "flashlight python bindings are required to use this functionality. " - "Please install from " - "https://github.com/facebookresearch/flashlight/tree/master/bindings/python" - ) - LM = object - LMState = object - - -class KenLMDecoder(BaseDecoder): - def __init__(self, cfg: FlashlightDecoderConfig, tgt_dict: Dictionary) -> None: - super().__init__(tgt_dict) - - self.nbest = cfg.nbest - self.unitlm = cfg.unitlm - - if cfg.lexicon: - self.lexicon = load_words(cfg.lexicon) - self.word_dict = create_word_dict(self.lexicon) - self.unk_word = self.word_dict.get_index("") - - self.lm = KenLM(cfg.lmpath, self.word_dict) - self.trie = Trie(self.vocab_size, self.silence) - - start_state = self.lm.start(False) - for word, spellings in self.lexicon.items(): - word_idx = self.word_dict.get_index(word) - _, score = self.lm.score(start_state, word_idx) - for spelling in spellings: - spelling_idxs = [tgt_dict.index(token) for token in spelling] - assert ( - tgt_dict.unk() not in spelling_idxs - ), f"{word} {spelling} {spelling_idxs}" - self.trie.insert(spelling_idxs, word_idx, score) - self.trie.smear(SmearingMode.MAX) - - self.decoder_opts = LexiconDecoderOptions( - beam_size=cfg.beam, - beam_size_token=cfg.beamsizetoken or len(tgt_dict), - beam_threshold=cfg.beamthreshold, - lm_weight=cfg.lmweight, - word_score=cfg.wordscore, - unk_score=cfg.unkweight, - sil_score=cfg.silweight, - log_add=False, - criterion_type=CriterionType.CTC, - ) - - self.decoder = LexiconDecoder( - self.decoder_opts, - self.trie, - self.lm, - self.silence, - self.blank, - self.unk_word, - [], - self.unitlm, - ) - else: - assert self.unitlm, "Lexicon-free decoding requires unit LM" - - d = {w: [[w]] for w in tgt_dict.symbols} - self.word_dict = create_word_dict(d) - self.lm = KenLM(cfg.lmpath, self.word_dict) - self.decoder_opts = LexiconFreeDecoderOptions( - beam_size=cfg.beam, - beam_size_token=cfg.beamsizetoken or len(tgt_dict), - beam_threshold=cfg.beamthreshold, - lm_weight=cfg.lmweight, - sil_score=cfg.silweight, - log_add=False, - criterion_type=CriterionType.CTC, - ) - self.decoder = LexiconFreeDecoder( - self.decoder_opts, self.lm, self.silence, self.blank, [] - ) - - def get_timesteps(self, token_idxs: List[int]) -> List[int]: - """Returns frame numbers corresponding to every non-blank token. - - Parameters - ---------- - token_idxs : List[int] - IDs of decoded tokens. - - Returns - ------- - List[int] - Frame numbers corresponding to every non-blank token. - """ - timesteps = [] - for i, token_idx in enumerate(token_idxs): - if token_idx == self.blank: - continue - if i == 0 or token_idx != token_idxs[i-1]: - timesteps.append(i) - return timesteps - - def decode( - self, - emissions: torch.FloatTensor, - ) -> List[List[Dict[str, torch.LongTensor]]]: - B, T, N = emissions.size() - hypos = [] - for b in range(B): - emissions_ptr = emissions.data_ptr() + 4 * b * emissions.stride(0) - results = self.decoder.decode(emissions_ptr, T, N) - - nbest_results = results[: self.nbest] - hypos.append( - [ - { - "tokens": self.get_tokens(result.tokens), - "score": result.score, - "timesteps": self.get_timesteps(result.tokens), - "words": [ - self.word_dict.get_entry(x) for x in result.words if x >= 0 - ], - } - for result in nbest_results - ] - ) - return hypos - - -FairseqLMState = namedtuple( - "FairseqLMState", - [ - "prefix", - "incremental_state", - "probs", - ], -) - - -class FairseqLM(LM): - def __init__(self, dictionary: Dictionary, model: FairseqModel) -> None: - super().__init__() - - self.dictionary = dictionary - self.model = model - self.unk = self.dictionary.unk() - - self.save_incremental = False # this currently does not work properly - self.max_cache = 20_000 - - if torch.cuda.is_available(): - model.cuda() - model.eval() - model.make_generation_fast_() - - self.states = {} - self.stateq = deque() - - def start(self, start_with_nothing: bool) -> LMState: - state = LMState() - prefix = torch.LongTensor([[self.dictionary.eos()]]) - incremental_state = {} if self.save_incremental else None - with torch.no_grad(): - res = self.model(prefix.cuda(), incremental_state=incremental_state) - probs = self.model.get_normalized_probs(res, log_probs=True, sample=None) - - if incremental_state is not None: - incremental_state = apply_to_sample(lambda x: x.cpu(), incremental_state) - self.states[state] = FairseqLMState( - prefix.numpy(), incremental_state, probs[0, -1].cpu().numpy() - ) - self.stateq.append(state) - - return state - - def score( - self, - state: LMState, - token_index: int, - no_cache: bool = False, - ) -> Tuple[LMState, int]: - """ - Evaluate language model based on the current lm state and new word - Parameters: - ----------- - state: current lm state - token_index: index of the word - (can be lexicon index then you should store inside LM the - mapping between indices of lexicon and lm, or lm index of a word) - Returns: - -------- - (LMState, float): pair of (new state, score for the current word) - """ - curr_state = self.states[state] - - def trim_cache(targ_size: int) -> None: - while len(self.stateq) > targ_size: - rem_k = self.stateq.popleft() - rem_st = self.states[rem_k] - rem_st = FairseqLMState(rem_st.prefix, None, None) - self.states[rem_k] = rem_st - - if curr_state.probs is None: - new_incremental_state = ( - curr_state.incremental_state.copy() - if curr_state.incremental_state is not None - else None - ) - with torch.no_grad(): - if new_incremental_state is not None: - new_incremental_state = apply_to_sample( - lambda x: x.cuda(), new_incremental_state - ) - elif self.save_incremental: - new_incremental_state = {} - - res = self.model( - torch.from_numpy(curr_state.prefix).cuda(), - incremental_state=new_incremental_state, - ) - probs = self.model.get_normalized_probs( - res, log_probs=True, sample=None - ) - - if new_incremental_state is not None: - new_incremental_state = apply_to_sample( - lambda x: x.cpu(), new_incremental_state - ) - - curr_state = FairseqLMState( - curr_state.prefix, new_incremental_state, probs[0, -1].cpu().numpy() - ) - - if not no_cache: - self.states[state] = curr_state - self.stateq.append(state) - - score = curr_state.probs[token_index].item() - - trim_cache(self.max_cache) - - outstate = state.child(token_index) - if outstate not in self.states and not no_cache: - prefix = np.concatenate( - [curr_state.prefix, torch.LongTensor([[token_index]])], -1 - ) - incr_state = curr_state.incremental_state - - self.states[outstate] = FairseqLMState(prefix, incr_state, None) - - if token_index == self.unk: - score = float("-inf") - - return outstate, score - - def finish(self, state: LMState) -> Tuple[LMState, int]: - """ - Evaluate eos for language model based on the current lm state - Returns: - -------- - (LMState, float): pair of (new state, score for the current word) - """ - return self.score(state, self.dictionary.eos()) - - def empty_cache(self) -> None: - self.states = {} - self.stateq = deque() - gc.collect() - - -class FairseqLMDecoder(BaseDecoder): - def __init__(self, cfg: FlashlightDecoderConfig, tgt_dict: Dictionary) -> None: - super().__init__(tgt_dict) - - self.nbest = cfg.nbest - self.unitlm = cfg.unitlm - - self.lexicon = load_words(cfg.lexicon) if cfg.lexicon else None - self.idx_to_wrd = {} - - checkpoint = torch.load(cfg.lmpath, map_location="cpu") - - if "cfg" in checkpoint and checkpoint["cfg"] is not None: - lm_args = checkpoint["cfg"] - else: - lm_args = convert_namespace_to_omegaconf(checkpoint["args"]) - - if not OmegaConf.is_dict(lm_args): - lm_args = OmegaConf.create(lm_args) - - with open_dict(lm_args.task): - lm_args.task.data = osp.dirname(cfg.lmpath) - - task = tasks.setup_task(lm_args.task) - model = task.build_model(lm_args.model) - model.load_state_dict(checkpoint["model"], strict=False) - - self.trie = Trie(self.vocab_size, self.silence) - - self.word_dict = task.dictionary - self.unk_word = self.word_dict.unk() - self.lm = FairseqLM(self.word_dict, model) - - if self.lexicon: - start_state = self.lm.start(False) - for i, (word, spellings) in enumerate(self.lexicon.items()): - if self.unitlm: - word_idx = i - self.idx_to_wrd[i] = word - score = 0 - else: - word_idx = self.word_dict.index(word) - _, score = self.lm.score(start_state, word_idx, no_cache=True) - - for spelling in spellings: - spelling_idxs = [tgt_dict.index(token) for token in spelling] - assert ( - tgt_dict.unk() not in spelling_idxs - ), f"{spelling} {spelling_idxs}" - self.trie.insert(spelling_idxs, word_idx, score) - self.trie.smear(SmearingMode.MAX) - - self.decoder_opts = LexiconDecoderOptions( - beam_size=cfg.beam, - beam_size_token=cfg.beamsizetoken or len(tgt_dict), - beam_threshold=cfg.beamthreshold, - lm_weight=cfg.lmweight, - word_score=cfg.wordscore, - unk_score=cfg.unkweight, - sil_score=cfg.silweight, - log_add=False, - criterion_type=CriterionType.CTC, - ) - - self.decoder = LexiconDecoder( - self.decoder_opts, - self.trie, - self.lm, - self.silence, - self.blank, - self.unk_word, - [], - self.unitlm, - ) - else: - assert self.unitlm, "Lexicon-free decoding requires unit LM" - - d = {w: [[w]] for w in tgt_dict.symbols} - self.word_dict = create_word_dict(d) - self.lm = KenLM(cfg.lmpath, self.word_dict) - self.decoder_opts = LexiconFreeDecoderOptions( - beam_size=cfg.beam, - beam_size_token=cfg.beamsizetoken or len(tgt_dict), - beam_threshold=cfg.beamthreshold, - lm_weight=cfg.lmweight, - sil_score=cfg.silweight, - log_add=False, - criterion_type=CriterionType.CTC, - ) - self.decoder = LexiconFreeDecoder( - self.decoder_opts, self.lm, self.silence, self.blank, [] - ) - - def decode( - self, - emissions: torch.FloatTensor, - ) -> List[List[Dict[str, torch.LongTensor]]]: - B, T, N = emissions.size() - hypos = [] - - def make_hypo(result: DecodeResult) -> Dict[str, Any]: - hypo = { - "tokens": self.get_tokens(result.tokens), - "score": result.score, - } - if self.lexicon: - hypo["words"] = [ - self.idx_to_wrd[x] if self.unitlm else self.word_dict[x] - for x in result.words - if x >= 0 - ] - return hypo - - for b in range(B): - emissions_ptr = emissions.data_ptr() + 4 * b * emissions.stride(0) - results = self.decoder.decode(emissions_ptr, T, N) - - nbest_results = results[: self.nbest] - hypos.append([make_hypo(result) for result in nbest_results]) - self.lm.empty_cache() - - return hypos diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/new/decoders/viterbi_decoder.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/new/decoders/viterbi_decoder.py deleted file mode 100644 index f24087056288a479f6ab62be40ccb6529b693408..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/new/decoders/viterbi_decoder.py +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from typing import Dict, List - -import torch - -from .base_decoder import BaseDecoder - - -class ViterbiDecoder(BaseDecoder): - def decode( - self, - emissions: torch.FloatTensor, - ) -> List[List[Dict[str, torch.LongTensor]]]: - def get_pred(e): - toks = e.argmax(dim=-1).unique_consecutive() - return toks[toks != self.blank] - - return [[{"tokens": get_pred(x), "score": 0}] for x in emissions] diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/new/infer.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/new/infer.py deleted file mode 100644 index 195f7102cd0256a593391db0d3c20f55c5ea4417..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/new/infer.py +++ /dev/null @@ -1,474 +0,0 @@ -#!/usr/bin/env python -u -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import ast -import hashlib -import logging -import os -import shutil -import sys -from dataclasses import dataclass, field, is_dataclass -from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple, Union - -import editdistance -import hydra -import torch -import torch.distributed as dist -from examples.speech_recognition.new.decoders.decoder import Decoder -from examples.speech_recognition.new.decoders.decoder_config import ( - DecoderConfig, - FlashlightDecoderConfig, -) -from fairseq import checkpoint_utils, distributed_utils, progress_bar, tasks, utils -from fairseq.data.data_utils import post_process -from fairseq.dataclass.configs import ( - CheckpointConfig, - CommonConfig, - CommonEvalConfig, - DatasetConfig, - DistributedTrainingConfig, - FairseqDataclass, -) -from fairseq.logging.meters import StopwatchMeter, TimeMeter -from fairseq.logging.progress_bar import BaseProgressBar -from fairseq.models.fairseq_model import FairseqModel -from hydra.core.config_store import ConfigStore -from omegaconf import OmegaConf - -logging.root.setLevel(logging.INFO) -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - -config_path = Path(__file__).resolve().parent / "conf" - - -@dataclass -class DecodingConfig(DecoderConfig, FlashlightDecoderConfig): - unique_wer_file: bool = field( - default=False, - metadata={"help": "If set, use a unique file for storing WER"}, - ) - results_path: Optional[str] = field( - default=None, - metadata={ - "help": "If set, write hypothesis and reference sentences into this directory" - }, - ) - - -@dataclass -class InferConfig(FairseqDataclass): - task: Any = None - decoding: DecodingConfig = DecodingConfig() - common: CommonConfig = CommonConfig() - common_eval: CommonEvalConfig = CommonEvalConfig() - checkpoint: CheckpointConfig = CheckpointConfig() - distributed_training: DistributedTrainingConfig = DistributedTrainingConfig() - dataset: DatasetConfig = DatasetConfig() - is_ax: bool = field( - default=False, - metadata={ - "help": "if true, assumes we are using ax for tuning and returns a tuple for ax to consume" - }, - ) - - -def reset_logging(): - root = logging.getLogger() - for handler in root.handlers: - root.removeHandler(handler) - root.setLevel(os.environ.get("LOGLEVEL", "INFO").upper()) - handler = logging.StreamHandler(sys.stdout) - handler.setFormatter( - logging.Formatter( - fmt="%(asctime)s | %(levelname)s | %(name)s | %(message)s", - datefmt="%Y-%m-%d %H:%M:%S", - ) - ) - root.addHandler(handler) - - -class InferenceProcessor: - cfg: InferConfig - - def __init__(self, cfg: InferConfig) -> None: - self.cfg = cfg - self.task = tasks.setup_task(cfg.task) - - models, saved_cfg = self.load_model_ensemble() - self.models = models - self.saved_cfg = saved_cfg - self.tgt_dict = self.task.target_dictionary - - self.task.load_dataset( - self.cfg.dataset.gen_subset, - task_cfg=saved_cfg.task, - ) - self.generator = Decoder(cfg.decoding, self.tgt_dict) - self.gen_timer = StopwatchMeter() - self.wps_meter = TimeMeter() - self.num_sentences = 0 - self.total_errors = 0 - self.total_length = 0 - - self.hypo_words_file = None - self.hypo_units_file = None - self.ref_words_file = None - self.ref_units_file = None - - self.progress_bar = self.build_progress_bar() - - def __enter__(self) -> "InferenceProcessor": - if self.cfg.decoding.results_path is not None: - self.hypo_words_file = self.get_res_file("hypo.word") - self.hypo_units_file = self.get_res_file("hypo.units") - self.ref_words_file = self.get_res_file("ref.word") - self.ref_units_file = self.get_res_file("ref.units") - return self - - def __exit__(self, *exc) -> bool: - if self.cfg.decoding.results_path is not None: - self.hypo_words_file.close() - self.hypo_units_file.close() - self.ref_words_file.close() - self.ref_units_file.close() - return False - - def __iter__(self) -> Any: - for sample in self.progress_bar: - if not self.cfg.common.cpu: - sample = utils.move_to_cuda(sample) - - # Happens on the last batch. - if "net_input" not in sample: - continue - yield sample - - def log(self, *args, **kwargs): - self.progress_bar.log(*args, **kwargs) - - def print(self, *args, **kwargs): - self.progress_bar.print(*args, **kwargs) - - def get_res_file(self, fname: str) -> None: - fname = os.path.join(self.cfg.decoding.results_path, fname) - if self.data_parallel_world_size > 1: - fname = f"{fname}.{self.data_parallel_rank}" - return open(fname, "w", buffering=1) - - def merge_shards(self) -> None: - """Merges all shard files into shard 0, then removes shard suffix.""" - - shard_id = self.data_parallel_rank - num_shards = self.data_parallel_world_size - - if self.data_parallel_world_size > 1: - - def merge_shards_with_root(fname: str) -> None: - fname = os.path.join(self.cfg.decoding.results_path, fname) - logger.info("Merging %s on shard %d", fname, shard_id) - base_fpath = Path(f"{fname}.0") - with open(base_fpath, "a") as out_file: - for s in range(1, num_shards): - shard_fpath = Path(f"{fname}.{s}") - with open(shard_fpath, "r") as in_file: - for line in in_file: - out_file.write(line) - shard_fpath.unlink() - shutil.move(f"{fname}.0", fname) - - dist.barrier() # ensure all shards finished writing - if shard_id == (0 % num_shards): - merge_shards_with_root("hypo.word") - if shard_id == (1 % num_shards): - merge_shards_with_root("hypo.units") - if shard_id == (2 % num_shards): - merge_shards_with_root("ref.word") - if shard_id == (3 % num_shards): - merge_shards_with_root("ref.units") - dist.barrier() - - def optimize_model(self, model: FairseqModel) -> None: - model.make_generation_fast_() - if self.cfg.common.fp16: - model.half() - if not self.cfg.common.cpu: - model.cuda() - - def load_model_ensemble(self) -> Tuple[List[FairseqModel], FairseqDataclass]: - arg_overrides = ast.literal_eval(self.cfg.common_eval.model_overrides) - models, saved_cfg = checkpoint_utils.load_model_ensemble( - utils.split_paths(self.cfg.common_eval.path, separator="\\"), - arg_overrides=arg_overrides, - task=self.task, - suffix=self.cfg.checkpoint.checkpoint_suffix, - strict=(self.cfg.checkpoint.checkpoint_shard_count == 1), - num_shards=self.cfg.checkpoint.checkpoint_shard_count, - ) - for model in models: - self.optimize_model(model) - return models, saved_cfg - - def get_dataset_itr(self, disable_iterator_cache: bool = False) -> None: - return self.task.get_batch_iterator( - dataset=self.task.dataset(self.cfg.dataset.gen_subset), - max_tokens=self.cfg.dataset.max_tokens, - max_sentences=self.cfg.dataset.batch_size, - max_positions=(sys.maxsize, sys.maxsize), - ignore_invalid_inputs=self.cfg.dataset.skip_invalid_size_inputs_valid_test, - required_batch_size_multiple=self.cfg.dataset.required_batch_size_multiple, - seed=self.cfg.common.seed, - num_shards=self.data_parallel_world_size, - shard_id=self.data_parallel_rank, - num_workers=self.cfg.dataset.num_workers, - data_buffer_size=self.cfg.dataset.data_buffer_size, - disable_iterator_cache=disable_iterator_cache, - ).next_epoch_itr(shuffle=False) - - def build_progress_bar( - self, - epoch: Optional[int] = None, - prefix: Optional[str] = None, - default_log_format: str = "tqdm", - ) -> BaseProgressBar: - return progress_bar.progress_bar( - iterator=self.get_dataset_itr(), - log_format=self.cfg.common.log_format, - log_interval=self.cfg.common.log_interval, - epoch=epoch, - prefix=prefix, - tensorboard_logdir=self.cfg.common.tensorboard_logdir, - default_log_format=default_log_format, - ) - - @property - def data_parallel_world_size(self): - if self.cfg.distributed_training.distributed_world_size == 1: - return 1 - return distributed_utils.get_data_parallel_world_size() - - @property - def data_parallel_rank(self): - if self.cfg.distributed_training.distributed_world_size == 1: - return 0 - return distributed_utils.get_data_parallel_rank() - - def process_sentence( - self, - sample: Dict[str, Any], - hypo: Dict[str, Any], - sid: int, - batch_id: int, - ) -> Tuple[int, int]: - speaker = None # Speaker can't be parsed from dataset. - - if "target_label" in sample: - toks = sample["target_label"] - else: - toks = sample["target"] - toks = toks[batch_id, :] - - # Processes hypothesis. - hyp_pieces = self.tgt_dict.string(hypo["tokens"].int().cpu()) - if "words" in hypo: - hyp_words = " ".join(hypo["words"]) - else: - hyp_words = post_process(hyp_pieces, self.cfg.common_eval.post_process) - - # Processes target. - target_tokens = utils.strip_pad(toks, self.tgt_dict.pad()) - tgt_pieces = self.tgt_dict.string(target_tokens.int().cpu()) - tgt_words = post_process(tgt_pieces, self.cfg.common_eval.post_process) - - if self.cfg.decoding.results_path is not None: - print(f"{hyp_pieces} ({speaker}-{sid})", file=self.hypo_units_file) - print(f"{hyp_words} ({speaker}-{sid})", file=self.hypo_words_file) - print(f"{tgt_pieces} ({speaker}-{sid})", file=self.ref_units_file) - print(f"{tgt_words} ({speaker}-{sid})", file=self.ref_words_file) - - if not self.cfg.common_eval.quiet: - logger.info(f"HYPO: {hyp_words}") - logger.info(f"REF: {tgt_words}") - logger.info("---------------------") - - hyp_words, tgt_words = hyp_words.split(), tgt_words.split() - - return editdistance.eval(hyp_words, tgt_words), len(tgt_words) - - def process_sample(self, sample: Dict[str, Any]) -> None: - self.gen_timer.start() - hypos = self.task.inference_step( - generator=self.generator, - models=self.models, - sample=sample, - ) - num_generated_tokens = sum(len(h[0]["tokens"]) for h in hypos) - self.gen_timer.stop(num_generated_tokens) - self.wps_meter.update(num_generated_tokens) - - for batch_id, sample_id in enumerate(sample["id"].tolist()): - errs, length = self.process_sentence( - sample=sample, - sid=sample_id, - batch_id=batch_id, - hypo=hypos[batch_id][0], - ) - self.total_errors += errs - self.total_length += length - - self.log({"wps": round(self.wps_meter.avg)}) - if "nsentences" in sample: - self.num_sentences += sample["nsentences"] - else: - self.num_sentences += sample["id"].numel() - - def log_generation_time(self) -> None: - logger.info( - "Processed %d sentences (%d tokens) in %.1fs %.2f " - "sentences per second, %.2f tokens per second)", - self.num_sentences, - self.gen_timer.n, - self.gen_timer.sum, - self.num_sentences / (self.gen_timer.sum + 1e-6), - 1.0 / (self.gen_timer.avg + 1e-6), - ) - - -def parse_wer(wer_file: Path) -> float: - with open(wer_file, "r") as f: - return float(f.readline().strip().split(" ")[1]) - - -def get_wer_file(cfg: InferConfig) -> Path: - """Hashes the decoding parameters to a unique file ID.""" - base_path = "wer" - if cfg.decoding.results_path is not None: - base_path = os.path.join(cfg.decoding.results_path, base_path) - - if cfg.decoding.unique_wer_file: - yaml_str = OmegaConf.to_yaml(cfg.decoding) - fid = int(hashlib.md5(yaml_str.encode("utf-8")).hexdigest(), 16) - return Path(f"{base_path}.{fid % 1000000}") - else: - return Path(base_path) - - -def main(cfg: InferConfig) -> float: - """Entry point for main processing logic. - - Args: - cfg: The inferance configuration to use. - wer: Optional shared memory pointer for returning the WER. If not None, - the final WER value will be written here instead of being returned. - - Returns: - The final WER if `wer` is None, otherwise None. - """ - - yaml_str, wer_file = OmegaConf.to_yaml(cfg.decoding), get_wer_file(cfg) - - # Validates the provided configuration. - if cfg.dataset.max_tokens is None and cfg.dataset.batch_size is None: - cfg.dataset.max_tokens = 4000000 - if not cfg.common.cpu and not torch.cuda.is_available(): - raise ValueError("CUDA not found; set `cpu=True` to run without CUDA") - - logger.info(cfg.common_eval.path) - - with InferenceProcessor(cfg) as processor: - for sample in processor: - processor.process_sample(sample) - - processor.log_generation_time() - - if cfg.decoding.results_path is not None: - processor.merge_shards() - - errs_t, leng_t = processor.total_errors, processor.total_length - - if cfg.common.cpu: - logger.warning("Merging WER requires CUDA.") - elif processor.data_parallel_world_size > 1: - stats = torch.LongTensor([errs_t, leng_t]).cuda() - dist.all_reduce(stats, op=dist.ReduceOp.SUM) - errs_t, leng_t = stats[0].item(), stats[1].item() - - wer = errs_t * 100.0 / leng_t - - if distributed_utils.is_master(cfg.distributed_training): - with open(wer_file, "w") as f: - f.write( - ( - f"WER: {wer}\n" - f"err / num_ref_words = {errs_t} / {leng_t}\n\n" - f"{yaml_str}" - ) - ) - - return wer - - -@hydra.main(config_path=config_path, config_name="infer") -def hydra_main(cfg: InferConfig) -> Union[float, Tuple[float, Optional[float]]]: - container = OmegaConf.to_container(cfg, resolve=True, enum_to_str=True) - cfg = OmegaConf.create(container) - OmegaConf.set_struct(cfg, True) - - if cfg.common.reset_logging: - reset_logging() - - utils.import_user_module(cfg.common) - - # logger.info("Config:\n%s", OmegaConf.to_yaml(cfg)) - wer = float("inf") - - try: - if cfg.common.profile: - with torch.cuda.profiler.profile(): - with torch.autograd.profiler.emit_nvtx(): - distributed_utils.call_main(cfg, main) - else: - distributed_utils.call_main(cfg, main) - - wer = parse_wer(get_wer_file(cfg)) - except BaseException as e: # pylint: disable=broad-except - if not cfg.common.suppress_crashes: - raise - else: - logger.error("Crashed! %s", str(e)) - - logger.info("Word error rate: %.4f", wer) - if cfg.is_ax: - return wer, None - - return wer - - -def cli_main() -> None: - try: - from hydra._internal.utils import ( # pylint: disable=import-outside-toplevel - get_args, - ) - - cfg_name = get_args().config_name or "infer" - except ImportError: - logger.warning("Failed to get config name from hydra args") - cfg_name = "infer" - - cs = ConfigStore.instance() - cs.store(name=cfg_name, node=InferConfig) - - for k in InferConfig.__dataclass_fields__: - if is_dataclass(InferConfig.__dataclass_fields__[k].type): - v = InferConfig.__dataclass_fields__[k].default - cs.store(name=k, node=v) - - hydra_main() # pylint: disable=no-value-for-parameter - - -if __name__ == "__main__": - cli_main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/tasks/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/tasks/__init__.py deleted file mode 100644 index a22d2ec9805103385c283fb255a3c6434aa82294..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/tasks/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -import importlib -import os - -for file in sorted(os.listdir(os.path.dirname(__file__))): - if file.endswith(".py") and not file.startswith("_"): - task_name = file[: file.find(".py")] - importlib.import_module("examples.speech_recognition.tasks." + task_name) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/tasks/speech_recognition.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/tasks/speech_recognition.py deleted file mode 100644 index d9f011d55ff4fdfeb4c04ca790c314d685708c3a..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/tasks/speech_recognition.py +++ /dev/null @@ -1,157 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import json -import os -import re -import sys - -import torch -from examples.speech_recognition.data import AsrDataset -from examples.speech_recognition.data.replabels import replabel_symbol -from fairseq.data import Dictionary -from fairseq.tasks import LegacyFairseqTask, register_task - - -def get_asr_dataset_from_json(data_json_path, tgt_dict): - """ - Parse data json and create dataset. - See scripts/asr_prep_json.py which pack json from raw files - - Json example: - { - "utts": { - "4771-29403-0025": { - "input": { - "length_ms": 170, - "path": "/tmp/file1.flac" - }, - "output": { - "text": "HELLO \n", - "token": "HE LLO", - "tokenid": "4815, 861" - } - }, - "1564-142299-0096": { - ... - } - } - """ - if not os.path.isfile(data_json_path): - raise FileNotFoundError("Dataset not found: {}".format(data_json_path)) - with open(data_json_path, "rb") as f: - data_samples = json.load(f)["utts"] - assert len(data_samples) != 0 - sorted_samples = sorted( - data_samples.items(), - key=lambda sample: int(sample[1]["input"]["length_ms"]), - reverse=True, - ) - aud_paths = [s[1]["input"]["path"] for s in sorted_samples] - ids = [s[0] for s in sorted_samples] - speakers = [] - for s in sorted_samples: - m = re.search("(.+?)-(.+?)-(.+?)", s[0]) - speakers.append(m.group(1) + "_" + m.group(2)) - frame_sizes = [s[1]["input"]["length_ms"] for s in sorted_samples] - tgt = [ - [int(i) for i in s[1]["output"]["tokenid"].split(", ")] - for s in sorted_samples - ] - # append eos - tgt = [[*t, tgt_dict.eos()] for t in tgt] - return AsrDataset(aud_paths, frame_sizes, tgt, tgt_dict, ids, speakers) - - -@register_task("speech_recognition") -class SpeechRecognitionTask(LegacyFairseqTask): - """ - Task for training speech recognition model. - """ - - @staticmethod - def add_args(parser): - """Add task-specific arguments to the parser.""" - parser.add_argument("data", help="path to data directory") - parser.add_argument( - "--silence-token", default="\u2581", help="token for silence (used by w2l)" - ) - parser.add_argument( - "--max-source-positions", - default=sys.maxsize, - type=int, - metavar="N", - help="max number of frames in the source sequence", - ) - parser.add_argument( - "--max-target-positions", - default=1024, - type=int, - metavar="N", - help="max number of tokens in the target sequence", - ) - - def __init__(self, args, tgt_dict): - super().__init__(args) - self.tgt_dict = tgt_dict - - @classmethod - def setup_task(cls, args, **kwargs): - """Setup the task (e.g., load dictionaries).""" - dict_path = os.path.join(args.data, "dict.txt") - if not os.path.isfile(dict_path): - raise FileNotFoundError("Dict not found: {}".format(dict_path)) - tgt_dict = Dictionary.load(dict_path) - - if args.criterion == "ctc_loss": - tgt_dict.add_symbol("") - elif args.criterion == "asg_loss": - for i in range(1, args.max_replabel + 1): - tgt_dict.add_symbol(replabel_symbol(i)) - - print("| dictionary: {} types".format(len(tgt_dict))) - return cls(args, tgt_dict) - - def load_dataset(self, split, combine=False, **kwargs): - """Load a given dataset split. - - Args: - split (str): name of the split (e.g., train, valid, test) - """ - data_json_path = os.path.join(self.args.data, "{}.json".format(split)) - self.datasets[split] = get_asr_dataset_from_json(data_json_path, self.tgt_dict) - - def build_generator(self, models, args, **unused): - w2l_decoder = getattr(args, "w2l_decoder", None) - if w2l_decoder == "viterbi": - from examples.speech_recognition.w2l_decoder import W2lViterbiDecoder - - return W2lViterbiDecoder(args, self.target_dictionary) - elif w2l_decoder == "kenlm": - from examples.speech_recognition.w2l_decoder import W2lKenLMDecoder - - return W2lKenLMDecoder(args, self.target_dictionary) - elif w2l_decoder == "fairseqlm": - from examples.speech_recognition.w2l_decoder import W2lFairseqLMDecoder - - return W2lFairseqLMDecoder(args, self.target_dictionary) - else: - return super().build_generator(models, args) - - @property - def target_dictionary(self): - """Return the :class:`~fairseq.data.Dictionary` for the language - model.""" - return self.tgt_dict - - @property - def source_dictionary(self): - """Return the source :class:`~fairseq.data.Dictionary` (if applicable - for this task).""" - return None - - def max_positions(self): - """Return the max speech and sentence length allowed by the task.""" - return (self.args.max_source_positions, self.args.max_target_positions) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/utils/wer_utils.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/utils/wer_utils.py deleted file mode 100644 index 6e77591c2b1950f16b1133649cc80f43b3b0c6b8..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/utils/wer_utils.py +++ /dev/null @@ -1,380 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from __future__ import absolute_import, division, print_function, unicode_literals - -import re -from collections import deque -from enum import Enum - -import numpy as np - -""" - Utility modules for computation of Word Error Rate, - Alignments, as well as more granular metrics like - deletion, insersion and substitutions. -""" - - -class Code(Enum): - match = 1 - substitution = 2 - insertion = 3 - deletion = 4 - - -class Token(object): - def __init__(self, lbl="", st=np.nan, en=np.nan): - if np.isnan(st): - self.label, self.start, self.end = "", 0.0, 0.0 - else: - self.label, self.start, self.end = lbl, st, en - - -class AlignmentResult(object): - def __init__(self, refs, hyps, codes, score): - self.refs = refs # std::deque - self.hyps = hyps # std::deque - self.codes = codes # std::deque - self.score = score # float - - -def coordinate_to_offset(row, col, ncols): - return int(row * ncols + col) - - -def offset_to_row(offset, ncols): - return int(offset / ncols) - - -def offset_to_col(offset, ncols): - return int(offset % ncols) - - -def trimWhitespace(str): - return re.sub(" +", " ", re.sub(" *$", "", re.sub("^ *", "", str))) - - -def str2toks(str): - pieces = trimWhitespace(str).split(" ") - toks = [] - for p in pieces: - toks.append(Token(p, 0.0, 0.0)) - return toks - - -class EditDistance(object): - def __init__(self, time_mediated): - self.time_mediated_ = time_mediated - self.scores_ = np.nan # Eigen::Matrix - self.backtraces_ = ( - np.nan - ) # Eigen::Matrix backtraces_; - self.confusion_pairs_ = {} - - def cost(self, ref, hyp, code): - if self.time_mediated_: - if code == Code.match: - return abs(ref.start - hyp.start) + abs(ref.end - hyp.end) - elif code == Code.insertion: - return hyp.end - hyp.start - elif code == Code.deletion: - return ref.end - ref.start - else: # substitution - return abs(ref.start - hyp.start) + abs(ref.end - hyp.end) + 0.1 - else: - if code == Code.match: - return 0 - elif code == Code.insertion or code == Code.deletion: - return 3 - else: # substitution - return 4 - - def get_result(self, refs, hyps): - res = AlignmentResult(refs=deque(), hyps=deque(), codes=deque(), score=np.nan) - - num_rows, num_cols = self.scores_.shape - res.score = self.scores_[num_rows - 1, num_cols - 1] - - curr_offset = coordinate_to_offset(num_rows - 1, num_cols - 1, num_cols) - - while curr_offset != 0: - curr_row = offset_to_row(curr_offset, num_cols) - curr_col = offset_to_col(curr_offset, num_cols) - - prev_offset = self.backtraces_[curr_row, curr_col] - - prev_row = offset_to_row(prev_offset, num_cols) - prev_col = offset_to_col(prev_offset, num_cols) - - res.refs.appendleft(curr_row - 1) # Note: this was .push_front() in C++ - res.hyps.appendleft(curr_col - 1) - if curr_row - 1 == prev_row and curr_col == prev_col: - res.codes.appendleft(Code.deletion) - elif curr_row == prev_row and curr_col - 1 == prev_col: - res.codes.appendleft(Code.insertion) - else: - # assert(curr_row - 1 == prev_row and curr_col - 1 == prev_col) - ref_str = refs[res.refs[0]].label - hyp_str = hyps[res.hyps[0]].label - - if ref_str == hyp_str: - res.codes.appendleft(Code.match) - else: - res.codes.appendleft(Code.substitution) - - confusion_pair = "%s -> %s" % (ref_str, hyp_str) - if confusion_pair not in self.confusion_pairs_: - self.confusion_pairs_[confusion_pair] = 1 - else: - self.confusion_pairs_[confusion_pair] += 1 - - curr_offset = prev_offset - - return res - - def align(self, refs, hyps): - if len(refs) == 0 and len(hyps) == 0: - return np.nan - - # NOTE: we're not resetting the values in these matrices because every value - # will be overridden in the loop below. If this assumption doesn't hold, - # be sure to set all entries in self.scores_ and self.backtraces_ to 0. - self.scores_ = np.zeros((len(refs) + 1, len(hyps) + 1)) - self.backtraces_ = np.zeros((len(refs) + 1, len(hyps) + 1)) - - num_rows, num_cols = self.scores_.shape - - for i in range(num_rows): - for j in range(num_cols): - if i == 0 and j == 0: - self.scores_[i, j] = 0.0 - self.backtraces_[i, j] = 0 - continue - - if i == 0: - self.scores_[i, j] = self.scores_[i, j - 1] + self.cost( - None, hyps[j - 1], Code.insertion - ) - self.backtraces_[i, j] = coordinate_to_offset(i, j - 1, num_cols) - continue - - if j == 0: - self.scores_[i, j] = self.scores_[i - 1, j] + self.cost( - refs[i - 1], None, Code.deletion - ) - self.backtraces_[i, j] = coordinate_to_offset(i - 1, j, num_cols) - continue - - # Below here both i and j are greater than 0 - ref = refs[i - 1] - hyp = hyps[j - 1] - best_score = self.scores_[i - 1, j - 1] + ( - self.cost(ref, hyp, Code.match) - if (ref.label == hyp.label) - else self.cost(ref, hyp, Code.substitution) - ) - - prev_row = i - 1 - prev_col = j - 1 - ins = self.scores_[i, j - 1] + self.cost(None, hyp, Code.insertion) - if ins < best_score: - best_score = ins - prev_row = i - prev_col = j - 1 - - delt = self.scores_[i - 1, j] + self.cost(ref, None, Code.deletion) - if delt < best_score: - best_score = delt - prev_row = i - 1 - prev_col = j - - self.scores_[i, j] = best_score - self.backtraces_[i, j] = coordinate_to_offset( - prev_row, prev_col, num_cols - ) - - return self.get_result(refs, hyps) - - -class WERTransformer(object): - def __init__(self, hyp_str, ref_str, verbose=True): - self.ed_ = EditDistance(False) - self.id2oracle_errs_ = {} - self.utts_ = 0 - self.words_ = 0 - self.insertions_ = 0 - self.deletions_ = 0 - self.substitutions_ = 0 - - self.process(["dummy_str", hyp_str, ref_str]) - - if verbose: - print("'%s' vs '%s'" % (hyp_str, ref_str)) - self.report_result() - - def process(self, input): # std::vector&& input - if len(input) < 3: - print( - "Input must be of the form ... , got ", - len(input), - " inputs:", - ) - return None - - # Align - # std::vector hyps; - # std::vector refs; - - hyps = str2toks(input[-2]) - refs = str2toks(input[-1]) - - alignment = self.ed_.align(refs, hyps) - if alignment is None: - print("Alignment is null") - return np.nan - - # Tally errors - ins = 0 - dels = 0 - subs = 0 - for code in alignment.codes: - if code == Code.substitution: - subs += 1 - elif code == Code.insertion: - ins += 1 - elif code == Code.deletion: - dels += 1 - - # Output - row = input - row.append(str(len(refs))) - row.append(str(ins)) - row.append(str(dels)) - row.append(str(subs)) - # print(row) - - # Accumulate - kIdIndex = 0 - kNBestSep = "/" - - pieces = input[kIdIndex].split(kNBestSep) - - if len(pieces) == 0: - print( - "Error splitting ", - input[kIdIndex], - " on '", - kNBestSep, - "', got empty list", - ) - return np.nan - - id = pieces[0] - if id not in self.id2oracle_errs_: - self.utts_ += 1 - self.words_ += len(refs) - self.insertions_ += ins - self.deletions_ += dels - self.substitutions_ += subs - self.id2oracle_errs_[id] = [ins, dels, subs] - else: - curr_err = ins + dels + subs - prev_err = np.sum(self.id2oracle_errs_[id]) - if curr_err < prev_err: - self.id2oracle_errs_[id] = [ins, dels, subs] - - return 0 - - def report_result(self): - # print("---------- Summary ---------------") - if self.words_ == 0: - print("No words counted") - return - - # 1-best - best_wer = ( - 100.0 - * (self.insertions_ + self.deletions_ + self.substitutions_) - / self.words_ - ) - - print( - "\tWER = %0.2f%% (%i utts, %i words, %0.2f%% ins, " - "%0.2f%% dels, %0.2f%% subs)" - % ( - best_wer, - self.utts_, - self.words_, - 100.0 * self.insertions_ / self.words_, - 100.0 * self.deletions_ / self.words_, - 100.0 * self.substitutions_ / self.words_, - ) - ) - - def wer(self): - if self.words_ == 0: - wer = np.nan - else: - wer = ( - 100.0 - * (self.insertions_ + self.deletions_ + self.substitutions_) - / self.words_ - ) - return wer - - def stats(self): - if self.words_ == 0: - stats = {} - else: - wer = ( - 100.0 - * (self.insertions_ + self.deletions_ + self.substitutions_) - / self.words_ - ) - stats = dict( - { - "wer": wer, - "utts": self.utts_, - "numwords": self.words_, - "ins": self.insertions_, - "dels": self.deletions_, - "subs": self.substitutions_, - "confusion_pairs": self.ed_.confusion_pairs_, - } - ) - return stats - - -def calc_wer(hyp_str, ref_str): - t = WERTransformer(hyp_str, ref_str, verbose=0) - return t.wer() - - -def calc_wer_stats(hyp_str, ref_str): - t = WERTransformer(hyp_str, ref_str, verbose=0) - return t.stats() - - -def get_wer_alignment_codes(hyp_str, ref_str): - """ - INPUT: hypothesis string, reference string - OUTPUT: List of alignment codes (intermediate results from WER computation) - """ - t = WERTransformer(hyp_str, ref_str, verbose=0) - return t.ed_.align(str2toks(ref_str), str2toks(hyp_str)).codes - - -def merge_counts(x, y): - # Merge two hashes which have 'counts' as their values - # This can be used for example to merge confusion pair counts - # conf_pairs = merge_counts(conf_pairs, stats['confusion_pairs']) - for k, v in y.items(): - if k not in x: - x[k] = 0 - x[k] += v - return x diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/w2l_decoder.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/w2l_decoder.py deleted file mode 100644 index 77de541a98252a1e7ae59725c6df9b6c5855d051..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/w2l_decoder.py +++ /dev/null @@ -1,491 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -""" -Flashlight decoders. -""" - -import gc -import itertools as it -import os.path as osp -import warnings -from collections import deque, namedtuple -from typing import List - -import numpy as np -import torch -from examples.speech_recognition.data.replabels import unpack_replabels -from fairseq import tasks -from fairseq.dataclass.utils import convert_namespace_to_omegaconf -from fairseq.utils import apply_to_sample -from omegaconf import open_dict - -try: - from flashlight.lib.sequence.criterion import CpuViterbiPath, get_data_ptr_as_bytes - from flashlight.lib.text.decoder import ( - LM, - CriterionType, - KenLM, - LexiconDecoder, - LexiconDecoderOptions, - LMState, - SmearingMode, - Trie, - ) - from flashlight.lib.text.dictionary import create_word_dict, load_words -except: - warnings.warn( - "flashlight python bindings are required to use this functionality. Please install from https://github.com/facebookresearch/flashlight/tree/master/bindings/python" - ) - LM = object - LMState = object - - -class W2lDecoder(object): - def __init__(self, args, tgt_dict): - self.tgt_dict = tgt_dict - self.vocab_size = len(tgt_dict) - self.nbest = args.nbest - - # criterion-specific init - self.criterion_type = CriterionType.CTC - self.blank = ( - tgt_dict.index("") - if "" in tgt_dict.indices - else tgt_dict.bos() - ) - if "" in tgt_dict.indices: - self.silence = tgt_dict.index("") - elif "|" in tgt_dict.indices: - self.silence = tgt_dict.index("|") - else: - self.silence = tgt_dict.eos() - self.asg_transitions = None - - def generate(self, models, sample, **unused): - """Generate a batch of inferences.""" - # model.forward normally channels prev_output_tokens into the decoder - # separately, but SequenceGenerator directly calls model.encoder - encoder_input = { - k: v for k, v in sample["net_input"].items() if k != "prev_output_tokens" - } - emissions = self.get_emissions(models, encoder_input) - return self.decode(emissions) - - def get_emissions(self, models, encoder_input): - """Run encoder and normalize emissions""" - model = models[0] - encoder_out = model(**encoder_input) - if hasattr(model, "get_logits"): - emissions = model.get_logits(encoder_out) # no need to normalize emissions - else: - emissions = model.get_normalized_probs(encoder_out, log_probs=True) - return emissions.transpose(0, 1).float().cpu().contiguous() - - def get_tokens(self, idxs): - """Normalize tokens by handling CTC blank, ASG replabels, etc.""" - idxs = (g[0] for g in it.groupby(idxs)) - idxs = filter(lambda x: x != self.blank, idxs) - return torch.LongTensor(list(idxs)) - - -class W2lViterbiDecoder(W2lDecoder): - def __init__(self, args, tgt_dict): - super().__init__(args, tgt_dict) - - def decode(self, emissions): - B, T, N = emissions.size() - hypos = [] - if self.asg_transitions is None: - transitions = torch.FloatTensor(N, N).zero_() - else: - transitions = torch.FloatTensor(self.asg_transitions).view(N, N) - viterbi_path = torch.IntTensor(B, T) - workspace = torch.ByteTensor(CpuViterbiPath.get_workspace_size(B, T, N)) - CpuViterbiPath.compute( - B, - T, - N, - get_data_ptr_as_bytes(emissions), - get_data_ptr_as_bytes(transitions), - get_data_ptr_as_bytes(viterbi_path), - get_data_ptr_as_bytes(workspace), - ) - return [ - [{"tokens": self.get_tokens(viterbi_path[b].tolist()), "score": 0}] - for b in range(B) - ] - - -class W2lKenLMDecoder(W2lDecoder): - def __init__(self, args, tgt_dict): - super().__init__(args, tgt_dict) - - self.unit_lm = getattr(args, "unit_lm", False) - - if args.lexicon: - self.lexicon = load_words(args.lexicon) - self.word_dict = create_word_dict(self.lexicon) - self.unk_word = self.word_dict.get_index("") - - self.lm = KenLM(args.kenlm_model, self.word_dict) - self.trie = Trie(self.vocab_size, self.silence) - - start_state = self.lm.start(False) - for i, (word, spellings) in enumerate(self.lexicon.items()): - word_idx = self.word_dict.get_index(word) - _, score = self.lm.score(start_state, word_idx) - for spelling in spellings: - spelling_idxs = [tgt_dict.index(token) for token in spelling] - assert ( - tgt_dict.unk() not in spelling_idxs - ), f"{spelling} {spelling_idxs}" - self.trie.insert(spelling_idxs, word_idx, score) - self.trie.smear(SmearingMode.MAX) - - self.decoder_opts = LexiconDecoderOptions( - beam_size=args.beam, - beam_size_token=int(getattr(args, "beam_size_token", len(tgt_dict))), - beam_threshold=args.beam_threshold, - lm_weight=args.lm_weight, - word_score=args.word_score, - unk_score=args.unk_weight, - sil_score=args.sil_weight, - log_add=False, - criterion_type=self.criterion_type, - ) - - if self.asg_transitions is None: - N = 768 - # self.asg_transitions = torch.FloatTensor(N, N).zero_() - self.asg_transitions = [] - - self.decoder = LexiconDecoder( - self.decoder_opts, - self.trie, - self.lm, - self.silence, - self.blank, - self.unk_word, - self.asg_transitions, - self.unit_lm, - ) - else: - assert args.unit_lm, "lexicon free decoding can only be done with a unit language model" - from flashlight.lib.text.decoder import ( - LexiconFreeDecoder, - LexiconFreeDecoderOptions, - ) - - d = {w: [[w]] for w in tgt_dict.symbols} - self.word_dict = create_word_dict(d) - self.lm = KenLM(args.kenlm_model, self.word_dict) - self.decoder_opts = LexiconFreeDecoderOptions( - beam_size=args.beam, - beam_size_token=int(getattr(args, "beam_size_token", len(tgt_dict))), - beam_threshold=args.beam_threshold, - lm_weight=args.lm_weight, - sil_score=args.sil_weight, - log_add=False, - criterion_type=self.criterion_type, - ) - self.decoder = LexiconFreeDecoder( - self.decoder_opts, self.lm, self.silence, self.blank, [] - ) - - def get_timesteps(self, token_idxs: List[int]) -> List[int]: - """Returns frame numbers corresponding to every non-blank token. - - Parameters - ---------- - token_idxs : List[int] - IDs of decoded tokens. - - Returns - ------- - List[int] - Frame numbers corresponding to every non-blank token. - """ - timesteps = [] - for i, token_idx in enumerate(token_idxs): - if token_idx == self.blank: - continue - if i == 0 or token_idx != token_idxs[i-1]: - timesteps.append(i) - return timesteps - - def decode(self, emissions): - B, T, N = emissions.size() - hypos = [] - for b in range(B): - emissions_ptr = emissions.data_ptr() + 4 * b * emissions.stride(0) - results = self.decoder.decode(emissions_ptr, T, N) - - nbest_results = results[: self.nbest] - hypos.append( - [ - { - "tokens": self.get_tokens(result.tokens), - "score": result.score, - "timesteps": self.get_timesteps(result.tokens), - "words": [ - self.word_dict.get_entry(x) for x in result.words if x >= 0 - ], - } - for result in nbest_results - ] - ) - return hypos - - -FairseqLMState = namedtuple("FairseqLMState", ["prefix", "incremental_state", "probs"]) - - -class FairseqLM(LM): - def __init__(self, dictionary, model): - LM.__init__(self) - self.dictionary = dictionary - self.model = model - self.unk = self.dictionary.unk() - - self.save_incremental = False # this currently does not work properly - self.max_cache = 20_000 - - model.cuda() - model.eval() - model.make_generation_fast_() - - self.states = {} - self.stateq = deque() - - def start(self, start_with_nothing): - state = LMState() - prefix = torch.LongTensor([[self.dictionary.eos()]]) - incremental_state = {} if self.save_incremental else None - with torch.no_grad(): - res = self.model(prefix.cuda(), incremental_state=incremental_state) - probs = self.model.get_normalized_probs(res, log_probs=True, sample=None) - - if incremental_state is not None: - incremental_state = apply_to_sample(lambda x: x.cpu(), incremental_state) - self.states[state] = FairseqLMState( - prefix.numpy(), incremental_state, probs[0, -1].cpu().numpy() - ) - self.stateq.append(state) - - return state - - def score(self, state: LMState, token_index: int, no_cache: bool = False): - """ - Evaluate language model based on the current lm state and new word - Parameters: - ----------- - state: current lm state - token_index: index of the word - (can be lexicon index then you should store inside LM the - mapping between indices of lexicon and lm, or lm index of a word) - - Returns: - -------- - (LMState, float): pair of (new state, score for the current word) - """ - curr_state = self.states[state] - - def trim_cache(targ_size): - while len(self.stateq) > targ_size: - rem_k = self.stateq.popleft() - rem_st = self.states[rem_k] - rem_st = FairseqLMState(rem_st.prefix, None, None) - self.states[rem_k] = rem_st - - if curr_state.probs is None: - new_incremental_state = ( - curr_state.incremental_state.copy() - if curr_state.incremental_state is not None - else None - ) - with torch.no_grad(): - if new_incremental_state is not None: - new_incremental_state = apply_to_sample( - lambda x: x.cuda(), new_incremental_state - ) - elif self.save_incremental: - new_incremental_state = {} - - res = self.model( - torch.from_numpy(curr_state.prefix).cuda(), - incremental_state=new_incremental_state, - ) - probs = self.model.get_normalized_probs( - res, log_probs=True, sample=None - ) - - if new_incremental_state is not None: - new_incremental_state = apply_to_sample( - lambda x: x.cpu(), new_incremental_state - ) - - curr_state = FairseqLMState( - curr_state.prefix, new_incremental_state, probs[0, -1].cpu().numpy() - ) - - if not no_cache: - self.states[state] = curr_state - self.stateq.append(state) - - score = curr_state.probs[token_index].item() - - trim_cache(self.max_cache) - - outstate = state.child(token_index) - if outstate not in self.states and not no_cache: - prefix = np.concatenate( - [curr_state.prefix, torch.LongTensor([[token_index]])], -1 - ) - incr_state = curr_state.incremental_state - - self.states[outstate] = FairseqLMState(prefix, incr_state, None) - - if token_index == self.unk: - score = float("-inf") - - return outstate, score - - def finish(self, state: LMState): - """ - Evaluate eos for language model based on the current lm state - - Returns: - -------- - (LMState, float): pair of (new state, score for the current word) - """ - return self.score(state, self.dictionary.eos()) - - def empty_cache(self): - self.states = {} - self.stateq = deque() - gc.collect() - - -class W2lFairseqLMDecoder(W2lDecoder): - def __init__(self, args, tgt_dict): - super().__init__(args, tgt_dict) - - self.unit_lm = getattr(args, "unit_lm", False) - - self.lexicon = load_words(args.lexicon) if args.lexicon else None - self.idx_to_wrd = {} - - checkpoint = torch.load(args.kenlm_model, map_location="cpu") - - if "cfg" in checkpoint and checkpoint["cfg"] is not None: - lm_args = checkpoint["cfg"] - else: - lm_args = convert_namespace_to_omegaconf(checkpoint["args"]) - - with open_dict(lm_args.task): - lm_args.task.data = osp.dirname(args.kenlm_model) - - task = tasks.setup_task(lm_args.task) - model = task.build_model(lm_args.model) - model.load_state_dict(checkpoint["model"], strict=False) - - self.trie = Trie(self.vocab_size, self.silence) - - self.word_dict = task.dictionary - self.unk_word = self.word_dict.unk() - self.lm = FairseqLM(self.word_dict, model) - - if self.lexicon: - start_state = self.lm.start(False) - for i, (word, spellings) in enumerate(self.lexicon.items()): - if self.unit_lm: - word_idx = i - self.idx_to_wrd[i] = word - score = 0 - else: - word_idx = self.word_dict.index(word) - _, score = self.lm.score(start_state, word_idx, no_cache=True) - - for spelling in spellings: - spelling_idxs = [tgt_dict.index(token) for token in spelling] - assert ( - tgt_dict.unk() not in spelling_idxs - ), f"{spelling} {spelling_idxs}" - self.trie.insert(spelling_idxs, word_idx, score) - self.trie.smear(SmearingMode.MAX) - - self.decoder_opts = LexiconDecoderOptions( - beam_size=args.beam, - beam_size_token=int(getattr(args, "beam_size_token", len(tgt_dict))), - beam_threshold=args.beam_threshold, - lm_weight=args.lm_weight, - word_score=args.word_score, - unk_score=args.unk_weight, - sil_score=args.sil_weight, - log_add=False, - criterion_type=self.criterion_type, - ) - - self.decoder = LexiconDecoder( - self.decoder_opts, - self.trie, - self.lm, - self.silence, - self.blank, - self.unk_word, - [], - self.unit_lm, - ) - else: - assert args.unit_lm, "lexicon free decoding can only be done with a unit language model" - from flashlight.lib.text.decoder import ( - LexiconFreeDecoder, - LexiconFreeDecoderOptions, - ) - - d = {w: [[w]] for w in tgt_dict.symbols} - self.word_dict = create_word_dict(d) - self.lm = KenLM(args.kenlm_model, self.word_dict) - self.decoder_opts = LexiconFreeDecoderOptions( - beam_size=args.beam, - beam_size_token=int(getattr(args, "beam_size_token", len(tgt_dict))), - beam_threshold=args.beam_threshold, - lm_weight=args.lm_weight, - sil_score=args.sil_weight, - log_add=False, - criterion_type=self.criterion_type, - ) - self.decoder = LexiconFreeDecoder( - self.decoder_opts, self.lm, self.silence, self.blank, [] - ) - - def decode(self, emissions): - B, T, N = emissions.size() - hypos = [] - - def idx_to_word(idx): - if self.unit_lm: - return self.idx_to_wrd[idx] - else: - return self.word_dict[idx] - - def make_hypo(result): - hypo = {"tokens": self.get_tokens(result.tokens), "score": result.score} - if self.lexicon: - hypo["words"] = [idx_to_word(x) for x in result.words if x >= 0] - return hypo - - for b in range(B): - emissions_ptr = emissions.data_ptr() + 4 * b * emissions.stride(0) - results = self.decoder.decode(emissions_ptr, T, N) - - nbest_results = results[: self.nbest] - hypos.append([make_hypo(result) for result in nbest_results]) - self.lm.empty_cache() - - return hypos diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/README.md b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/README.md deleted file mode 100644 index a31e7f68bd3128494d70c0a4dfa75b9d54aa4288..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/README.md +++ /dev/null @@ -1,38 +0,0 @@ -Speech Synthesis (S^2) -=== -[https://arxiv.org/abs/2109.06912](https://arxiv.org/abs/2109.06912) - -Speech synthesis with fairseq. - -## Features - -- Autoregressive and non-autoregressive models -- Multi-speaker synthesis -- Audio preprocessing (denoising, VAD, etc.) for less curated data -- Automatic metrics for model development -- Similar data configuration as [S2T](../speech_to_text/README.md) - - -## Examples -- [Single-speaker synthesis on LJSpeech](docs/ljspeech_example.md) -- [Multi-speaker synthesis on VCTK](docs/vctk_example.md) -- [Multi-speaker synthesis on Common Voice](docs/common_voice_example.md) - - -## Citation -Please cite as: -``` -@article{wang2021fairseqs2, - title={fairseq S\^{} 2: A Scalable and Integrable Speech Synthesis Toolkit}, - author={Wang, Changhan and Hsu, Wei-Ning and Adi, Yossi and Polyak, Adam and Lee, Ann and Chen, Peng-Jen and Gu, Jiatao and Pino, Juan}, - journal={arXiv preprint arXiv:2109.06912}, - year={2021} -} - -@inproceedings{ott2019fairseq, - title = {fairseq: A Fast, Extensible Toolkit for Sequence Modeling}, - author = {Myle Ott and Sergey Edunov and Alexei Baevski and Angela Fan and Sam Gross and Nathan Ng and David Grangier and Michael Auli}, - booktitle = {Proceedings of NAACL-HLT 2019: Demonstrations}, - year = {2019}, -} -``` diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/__init__.py deleted file mode 100644 index 6264236915a7269a4d920ee8213004374dd86a9a..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/data_utils.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/data_utils.py deleted file mode 100644 index bba7b2e4552dbef205e1c7543a0dd4f715264bad..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/data_utils.py +++ /dev/null @@ -1,347 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import io -import os -import tempfile -import zipfile -from dataclasses import dataclass -from itertools import groupby -from pathlib import Path -from typing import Dict, List, Optional - -import numpy as np -import torch -import torch.nn.functional as F -from examples.speech_to_text.data_utils import load_tsv_to_dicts -from fairseq.data.audio.audio_utils import ( - TTSMelScale, - TTSSpectrogram, - is_npy_data, - parse_path, - read_from_stored_zip, -) -from tqdm import tqdm - - -def trim_or_pad_to_target_length( - data_1d_or_2d: np.ndarray, target_length: int -) -> np.ndarray: - assert len(data_1d_or_2d.shape) in {1, 2} - delta = data_1d_or_2d.shape[0] - target_length - if delta >= 0: # trim if being longer - data_1d_or_2d = data_1d_or_2d[: target_length] - else: # pad if being shorter - if len(data_1d_or_2d.shape) == 1: - data_1d_or_2d = np.concatenate( - [data_1d_or_2d, np.zeros(-delta)], axis=0 - ) - else: - data_1d_or_2d = np.concatenate( - [data_1d_or_2d, np.zeros((-delta, data_1d_or_2d.shape[1]))], - axis=0 - ) - return data_1d_or_2d - - -def extract_logmel_spectrogram( - waveform: torch.Tensor, sample_rate: int, - output_path: Optional[Path] = None, win_length: int = 1024, - hop_length: int = 256, n_fft: int = 1024, - win_fn: callable = torch.hann_window, n_mels: int = 80, - f_min: float = 0., f_max: float = 8000, eps: float = 1e-5, - overwrite: bool = False, target_length: Optional[int] = None -): - if output_path is not None and output_path.is_file() and not overwrite: - return - - spectrogram_transform = TTSSpectrogram( - n_fft=n_fft, win_length=win_length, hop_length=hop_length, - window_fn=win_fn - ) - mel_scale_transform = TTSMelScale( - n_mels=n_mels, sample_rate=sample_rate, f_min=f_min, f_max=f_max, - n_stft=n_fft // 2 + 1 - ) - spectrogram = spectrogram_transform(waveform) - mel_spec = mel_scale_transform(spectrogram) - logmel_spec = torch.clamp(mel_spec, min=eps).log() - assert len(logmel_spec.shape) == 3 and logmel_spec.shape[0] == 1 - logmel_spec = logmel_spec.squeeze().t() # D x T -> T x D - if target_length is not None: - logmel_spec = trim_or_pad_to_target_length(logmel_spec, target_length) - - if output_path is not None: - np.save(output_path.as_posix(), logmel_spec) - else: - return logmel_spec - - -def extract_pitch( - waveform: torch.Tensor, sample_rate: int, - output_path: Optional[Path] = None, hop_length: int = 256, - log_scale: bool = True, phoneme_durations: Optional[List[int]] = None -): - if output_path is not None and output_path.is_file(): - return - - try: - import pyworld - except ImportError: - raise ImportError("Please install PyWORLD: pip install pyworld") - - _waveform = waveform.squeeze(0).double().numpy() - pitch, t = pyworld.dio( - _waveform, sample_rate, frame_period=hop_length / sample_rate * 1000 - ) - pitch = pyworld.stonemask(_waveform, pitch, t, sample_rate) - - if phoneme_durations is not None: - pitch = trim_or_pad_to_target_length(pitch, sum(phoneme_durations)) - try: - from scipy.interpolate import interp1d - except ImportError: - raise ImportError("Please install SciPy: pip install scipy") - nonzero_ids = np.where(pitch != 0)[0] - if len(nonzero_ids) == 0: - print((f"{output_path} has all empty values in the pitch contour")) - return - elif len(nonzero_ids) == 1: - print((f"{output_path} has only one non-zero values in the pitch contour")) - return - else: - interp_fn = interp1d( - nonzero_ids, - pitch[nonzero_ids], - fill_value=(pitch[nonzero_ids[0]], pitch[nonzero_ids[-1]]), - bounds_error=False, - ) - pitch = interp_fn(np.arange(0, len(pitch))) - d_cumsum = np.cumsum(np.concatenate([np.array([0]), phoneme_durations])) - pitch = np.array( - [ - np.mean(pitch[d_cumsum[i-1]: d_cumsum[i]]) - for i in range(1, len(d_cumsum)) - ] - ) - assert len(pitch) == len(phoneme_durations) - - if log_scale: - pitch = np.log(pitch + 1) - - if output_path is not None: - np.save(output_path.as_posix(), pitch) - else: - return pitch - - -def extract_energy( - waveform: torch.Tensor, output_path: Optional[Path] = None, - hop_length: int = 256, n_fft: int = 1024, log_scale: bool = True, - phoneme_durations: Optional[List[int]] = None -): - if output_path is not None and output_path.is_file(): - return - - assert len(waveform.shape) == 2 and waveform.shape[0] == 1 - waveform = waveform.view(1, 1, waveform.shape[1]) - waveform = F.pad( - waveform.unsqueeze(1), [n_fft // 2, n_fft // 2, 0, 0], - mode="reflect" - ) - waveform = waveform.squeeze(1) - - fourier_basis = np.fft.fft(np.eye(n_fft)) - cutoff = int((n_fft / 2 + 1)) - fourier_basis = np.vstack( - [np.real(fourier_basis[:cutoff, :]), - np.imag(fourier_basis[:cutoff, :])] - ) - - forward_basis = torch.FloatTensor(fourier_basis[:, None, :]) - forward_transform = F.conv1d( - waveform, forward_basis, stride=hop_length, padding=0 - ) - - real_part = forward_transform[:, :cutoff, :] - imag_part = forward_transform[:, cutoff:, :] - magnitude = torch.sqrt(real_part ** 2 + imag_part ** 2) - energy = torch.norm(magnitude, dim=1).squeeze(0).numpy() - - if phoneme_durations is not None: - energy = trim_or_pad_to_target_length(energy, sum(phoneme_durations)) - d_cumsum = np.cumsum(np.concatenate([np.array([0]), phoneme_durations])) - energy = np.array( - [ - np.mean(energy[d_cumsum[i - 1]: d_cumsum[i]]) - for i in range(1, len(d_cumsum)) - ] - ) - assert len(energy) == len(phoneme_durations) - - if log_scale: - energy = np.log(energy + 1) - - if output_path is not None: - np.save(output_path.as_posix(), energy) - else: - return energy - - -def get_global_cmvn(feature_root: Path, output_path: Optional[Path] = None): - mean_x, mean_x2, n_frames = None, None, 0 - feature_paths = feature_root.glob("*.npy") - for p in tqdm(feature_paths): - with open(p, 'rb') as f: - frames = np.load(f).squeeze() - - n_frames += frames.shape[0] - - cur_mean_x = frames.sum(axis=0) - if mean_x is None: - mean_x = cur_mean_x - else: - mean_x += cur_mean_x - - cur_mean_x2 = (frames ** 2).sum(axis=0) - if mean_x2 is None: - mean_x2 = cur_mean_x2 - else: - mean_x2 += cur_mean_x2 - - mean_x /= n_frames - mean_x2 /= n_frames - var_x = mean_x2 - mean_x ** 2 - std_x = np.sqrt(np.maximum(var_x, 1e-10)) - - if output_path is not None: - with open(output_path, 'wb') as f: - np.savez(f, mean=mean_x, std=std_x) - else: - return {"mean": mean_x, "std": std_x} - - -def ipa_phonemize(text, lang="en-us", use_g2p=False): - if use_g2p: - assert lang == "en-us", "g2pE phonemizer only works for en-us" - try: - from g2p_en import G2p - g2p = G2p() - return " ".join("|" if p == " " else p for p in g2p(text)) - except ImportError: - raise ImportError( - "Please install phonemizer: pip install g2p_en" - ) - else: - try: - from phonemizer import phonemize - from phonemizer.separator import Separator - return phonemize( - text, backend='espeak', language=lang, - separator=Separator(word="| ", phone=" ") - ) - except ImportError: - raise ImportError( - "Please install phonemizer: pip install phonemizer" - ) - - -@dataclass -class ForceAlignmentInfo(object): - tokens: List[str] - frame_durations: List[int] - start_sec: Optional[float] - end_sec: Optional[float] - - -def get_mfa_alignment_by_sample_id( - textgrid_zip_path: str, sample_id: str, sample_rate: int, - hop_length: int, silence_phones: List[str] = ("sil", "sp", "spn") -) -> ForceAlignmentInfo: - try: - import tgt - except ImportError: - raise ImportError("Please install TextGridTools: pip install tgt") - - filename = f"{sample_id}.TextGrid" - out_root = Path(tempfile.gettempdir()) - tgt_path = out_root / filename - with zipfile.ZipFile(textgrid_zip_path) as f_zip: - f_zip.extract(filename, path=out_root) - textgrid = tgt.io.read_textgrid(tgt_path.as_posix()) - os.remove(tgt_path) - - phones, frame_durations = [], [] - start_sec, end_sec, end_idx = 0, 0, 0 - for t in textgrid.get_tier_by_name("phones")._objects: - s, e, p = t.start_time, t.end_time, t.text - # Trim leading silences - if len(phones) == 0: - if p in silence_phones: - continue - else: - start_sec = s - phones.append(p) - if p not in silence_phones: - end_sec = e - end_idx = len(phones) - r = sample_rate / hop_length - frame_durations.append(int(np.round(e * r) - np.round(s * r))) - # Trim tailing silences - phones = phones[:end_idx] - frame_durations = frame_durations[:end_idx] - - return ForceAlignmentInfo( - tokens=phones, frame_durations=frame_durations, start_sec=start_sec, - end_sec=end_sec - ) - - -def get_mfa_alignment( - textgrid_zip_path: str, sample_ids: List[str], sample_rate: int, - hop_length: int -) -> Dict[str, ForceAlignmentInfo]: - return { - i: get_mfa_alignment_by_sample_id( - textgrid_zip_path, i, sample_rate, hop_length - ) for i in tqdm(sample_ids) - } - - -def get_unit_alignment( - id_to_unit_tsv_path: str, sample_ids: List[str] -) -> Dict[str, ForceAlignmentInfo]: - id_to_units = { - e["id"]: e["units"] for e in load_tsv_to_dicts(id_to_unit_tsv_path) - } - id_to_units = {i: id_to_units[i].split() for i in sample_ids} - id_to_units_collapsed = { - i: [uu for uu, _ in groupby(u)] for i, u in id_to_units.items() - } - id_to_durations = { - i: [len(list(g)) for _, g in groupby(u)] for i, u in id_to_units.items() - } - - return { - i: ForceAlignmentInfo( - tokens=id_to_units_collapsed[i], frame_durations=id_to_durations[i], - start_sec=None, end_sec=None - ) - for i in sample_ids - } - - -def get_feature_value_min_max(feature_paths: List[str]): - v_min, v_max = 1e-8, -1e-8 - for p in tqdm(feature_paths): - _path, slice_ptr = parse_path(p) - assert len(slice_ptr) == 2 - byte_data = read_from_stored_zip(_path, slice_ptr[0], slice_ptr[1]) - assert is_npy_data(byte_data) - path_or_fp = io.BytesIO(byte_data) - features = np.load(path_or_fp).squeeze() - v_min = min(v_min, features.min().item()) - v_max = max(v_max, features.max().item()) - return v_min, v_max diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/docs/common_voice_example.md b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/docs/common_voice_example.md deleted file mode 100644 index 1c0eef69a0adcfb60e6d89c6df63311fbb1eb4aa..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/docs/common_voice_example.md +++ /dev/null @@ -1,67 +0,0 @@ -[[Back]](..) - -# Common Voice - -[Common Voice](https://commonvoice.mozilla.org/en/datasets) is a public domain speech corpus with 11.2K hours of read -speech in 76 languages (the latest version 7.0). We provide examples for building -[Transformer](https://arxiv.org/abs/1809.08895) models on this dataset. - - -## Data preparation -[Download](https://commonvoice.mozilla.org/en/datasets) and unpack Common Voice v4 to a path `${DATA_ROOT}/${LANG_ID}`. -Create splits and generate audio manifests with -```bash -python -m examples.speech_synthesis.preprocessing.get_common_voice_audio_manifest \ - --data-root ${DATA_ROOT} \ - --lang ${LANG_ID} \ - --output-manifest-root ${AUDIO_MANIFEST_ROOT} --convert-to-wav -``` - -To denoise audio and trim leading/trailing silence using signal processing based VAD, run -```bash -for SPLIT in dev test train; do - python -m examples.speech_synthesis.preprocessing.denoise_and_vad_audio \ - --audio-manifest ${AUDIO_MANIFEST_ROOT}/${SPLIT}.audio.tsv \ - --output-dir ${PROCESSED_DATA_ROOT} \ - --denoise --vad --vad-agg-level 2 -done -``` - -which generates a new audio TSV manifest under `${PROCESSED_DATA_ROOT}` with updated path to the processed audio and -a new column for SNR. - -To do filtering by CER, follow the [Automatic Evaluation](../docs/ljspeech_example.md#automatic-evaluation) section to -run ASR model (add `--eval-target` to `get_eval_manifest` for evaluation on the reference audio; add `--err-unit char` -to `eval_asr` to compute CER instead of WER). The example-level CER is saved to -`${EVAL_OUTPUT_ROOT}/uer_cer.${SPLIT}.tsv`. - -Then, extract log-Mel spectrograms, generate feature manifest and create data configuration YAML with -```bash -python -m examples.speech_synthesis.preprocessing.get_feature_manifest \ - --audio-manifest-root ${AUDIO_MANIFEST_ROOT} \ - --output-root ${FEATURE_MANIFEST_ROOT} \ - --ipa-vocab --lang ${LANG_ID} \ - --snr-threshold 15 \ - --cer-threshold 0.1 --cer-tsv-path ${EVAL_OUTPUT_ROOT}/uer_cer.${SPLIT}.tsv -``` -where we use phoneme inputs (`--ipa-vocab`) as example. For sample filtering, we set the SNR and CER threshold -to 15 and 10%, respectively. - - -## Training -(Please refer to [the LJSpeech example](../docs/ljspeech_example.md#transformer).) - - -## Inference -(Please refer to [the LJSpeech example](../docs/ljspeech_example.md#inference).) - -## Automatic Evaluation -(Please refer to [the LJSpeech example](../docs/ljspeech_example.md#automatic-evaluation).) - -## Results - -| Language | Speakers | --arch | Params | Test MCD | Model | -|---|---|---|---|---|---| -| English | 200 | tts_transformer | 54M | 3.8 | [Download](https://dl.fbaipublicfiles.com/fairseq/s2/cv4_en200_transformer_phn.tar) | - -[[Back]](..) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/docs/ljspeech_example.md b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/docs/ljspeech_example.md deleted file mode 100644 index 836c30d6d5fbd0af3f57f58e903a579612587f25..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/docs/ljspeech_example.md +++ /dev/null @@ -1,137 +0,0 @@ -[[Back]](..) - -# LJSpeech - -[LJSpeech](https://keithito.com/LJ-Speech-Dataset) is a public domain TTS -corpus with around 24 hours of English speech sampled at 22.05kHz. We provide examples for building -[Transformer](https://arxiv.org/abs/1809.08895) and [FastSpeech 2](https://arxiv.org/abs/2006.04558) -models on this dataset. - - -## Data preparation - -Download data, create splits and generate audio manifests with -```bash -python -m examples.speech_synthesis.preprocessing.get_ljspeech_audio_manifest \ - --output-data-root ${AUDIO_DATA_ROOT} \ - --output-manifest-root ${AUDIO_MANIFEST_ROOT} -``` - -Then, extract log-Mel spectrograms, generate feature manifest and create data configuration YAML with -```bash -python -m examples.speech_synthesis.preprocessing.get_feature_manifest \ - --audio-manifest-root ${AUDIO_MANIFEST_ROOT} \ - --output-root ${FEATURE_MANIFEST_ROOT} \ - --ipa-vocab --use-g2p -``` -where we use phoneme inputs (`--ipa-vocab --use-g2p`) as example. - -FastSpeech 2 additionally requires frame durations, pitch and energy as auxiliary training targets. -Add `--add-fastspeech-targets` to include these fields in the feature manifests. We get frame durations either from -phoneme-level force-alignment or frame-level pseudo-text unit sequence. They should be pre-computed and specified via: -- `--textgrid-zip ${TEXT_GRID_ZIP_PATH}` for a ZIP file, inside which there is one - [TextGrid](https://www.fon.hum.uva.nl/praat/manual/TextGrid.html) file per sample to provide force-alignment info. -- `--id-to-units-tsv ${ID_TO_UNIT_TSV}` for a TSV file, where there are 2 columns for sample ID and - space-delimited pseudo-text unit sequence, respectively. - -For your convenience, we provide pre-computed -[force-alignment](https://dl.fbaipublicfiles.com/fairseq/s2/ljspeech_mfa.zip) from -[Montreal Forced Aligner](https://github.com/MontrealCorpusTools/Montreal-Forced-Aligner) and -[pseudo-text units](s3://dl.fbaipublicfiles.com/fairseq/s2/ljspeech_hubert.tsv) from -[HuBERT](https://github.com/pytorch/fairseq/tree/main/examples/hubert). You can also generate them by yourself using -a different software or model. - - -## Training -#### Transformer -```bash -fairseq-train ${FEATURE_MANIFEST_ROOT} --save-dir ${SAVE_DIR} \ - --config-yaml config.yaml --train-subset train --valid-subset dev \ - --num-workers 4 --max-tokens 30000 --max-update 200000 \ - --task text_to_speech --criterion tacotron2 --arch tts_transformer \ - --clip-norm 5.0 --n-frames-per-step 4 --bce-pos-weight 5.0 \ - --dropout 0.1 --attention-dropout 0.1 --activation-dropout 0.1 \ - --encoder-normalize-before --decoder-normalize-before \ - --optimizer adam --lr 2e-3 --lr-scheduler inverse_sqrt --warmup-updates 4000 \ - --seed 1 --update-freq 8 --eval-inference --best-checkpoint-metric mcd_loss -``` -where `SAVE_DIR` is the checkpoint root path. We set `--update-freq 8` to simulate 8 GPUs with 1 GPU. You may want to -update it accordingly when using more than 1 GPU. - -#### FastSpeech2 -```bash -fairseq-train ${FEATURE_MANIFEST_ROOT} --save-dir ${SAVE_DIR} \ - --config-yaml config.yaml --train-subset train --valid-subset dev \ - --num-workers 4 --max-sentences 6 --max-update 200000 \ - --task text_to_speech --criterion fastspeech2 --arch fastspeech2 \ - --clip-norm 5.0 --n-frames-per-step 1 \ - --dropout 0.1 --attention-dropout 0.1 \ - --optimizer adam --lr 5e-4 --lr-scheduler inverse_sqrt --warmup-updates 4000 \ - --seed 1 --update-freq 8 --eval-inference --best-checkpoint-metric mcd_loss -``` - - -## Inference -Average the last 5 checkpoints, generate the test split spectrogram and waveform using the default Griffin-Lim vocoder: -```bash -SPLIT=test -CHECKPOINT_NAME=avg_last_5 -CHECKPOINT_PATH=${SAVE_DIR}/checkpoint_${CHECKPOINT_NAME}.pt -python scripts/average_checkpoints.py --inputs ${SAVE_DIR} \ - --num-epoch-checkpoints 5 \ - --output ${CHECKPOINT_PATH} - -python -m examples.speech_synthesis.generate_waveform ${FEATURE_MANIFEST_ROOT} \ - --config-yaml config.yaml --gen-subset ${SPLIT} --task text_to_speech \ - --path ${CHECKPOINT_PATH} --max-tokens 50000 --spec-bwd-max-iter 32 \ - --dump-waveforms -``` -which dumps files (waveform, feature, attention plot, etc.) to `${SAVE_DIR}/generate-${CHECKPOINT_NAME}-${SPLIT}`. To -re-synthesize target waveforms for automatic evaluation, add `--dump-target`. - -## Automatic Evaluation -To start with, generate the manifest for synthetic speech, which will be taken as inputs by evaluation scripts. -```bash -python -m examples.speech_synthesis.evaluation.get_eval_manifest \ - --generation-root ${SAVE_DIR}/generate-${CHECKPOINT_NAME}-${SPLIT} \ - --audio-manifest ${AUDIO_MANIFEST_ROOT}/${SPLIT}.audio.tsv \ - --output-path ${EVAL_OUTPUT_ROOT}/eval.tsv \ - --vocoder griffin_lim --sample-rate 22050 --audio-format flac \ - --use-resynthesized-target -``` -Speech recognition (ASR) models usually operate at lower sample rates (e.g. 16kHz). For the WER/CER metric, -you may need to resample the audios accordingly --- add `--output-sample-rate 16000` for `generate_waveform.py` and -use `--sample-rate 16000` for `get_eval_manifest.py`. - - -#### WER/CER metric -We use wav2vec 2.0 ASR model as example. [Download](https://github.com/pytorch/fairseq/tree/main/examples/wav2vec) -the model checkpoint and dictionary, then compute WER/CER with -```bash -python -m examples.speech_synthesis.evaluation.eval_asr \ - --audio-header syn --text-header text --err-unit char --split ${SPLIT} \ - --w2v-ckpt ${WAV2VEC2_CHECKPOINT_PATH} --w2v-dict-dir ${WAV2VEC2_DICT_DIR} \ - --raw-manifest ${EVAL_OUTPUT_ROOT}/eval_16khz.tsv --asr-dir ${EVAL_OUTPUT_ROOT}/asr -``` - -#### MCD/MSD metric -```bash -python -m examples.speech_synthesis.evaluation.eval_sp \ - ${EVAL_OUTPUT_ROOT}/eval.tsv --mcd --msd -``` - -#### F0 metrics -```bash -python -m examples.speech_synthesis.evaluation.eval_f0 \ - ${EVAL_OUTPUT_ROOT}/eval.tsv --gpe --vde --ffe -``` - - -## Results - -| --arch | Params | Test MCD | Model | -|---|---|---|---| -| tts_transformer | 54M | 3.8 | [Download](https://dl.fbaipublicfiles.com/fairseq/s2/ljspeech_transformer_phn.tar) | -| fastspeech2 | 41M | 3.8 | [Download](https://dl.fbaipublicfiles.com/fairseq/s2/ljspeech_fastspeech2_phn.tar) | - -[[Back]](..) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/docs/vctk_example.md b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/docs/vctk_example.md deleted file mode 100644 index 6808256d44ef08b0350aa4d74fa062673990137a..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/docs/vctk_example.md +++ /dev/null @@ -1,61 +0,0 @@ -[[Back]](..) - -# VCTK - -[VCTK](https://datashare.ed.ac.uk/handle/10283/3443) is an open English speech corpus. We provide examples -for building [Transformer](https://arxiv.org/abs/1809.08895) models on this dataset. - - -## Data preparation -Download data, create splits and generate audio manifests with -```bash -python -m examples.speech_synthesis.preprocessing.get_vctk_audio_manifest \ - --output-data-root ${AUDIO_DATA_ROOT} \ - --output-manifest-root ${AUDIO_MANIFEST_ROOT} -``` - -To denoise audio and trim leading/trailing silence using signal processing based VAD, run -```bash -for SPLIT in dev test train; do - python -m examples.speech_synthesis.preprocessing.denoise_and_vad_audio \ - --audio-manifest ${AUDIO_MANIFEST_ROOT}/${SPLIT}.audio.tsv \ - --output-dir ${PROCESSED_DATA_ROOT} \ - --denoise --vad --vad-agg-level 3 -done -``` -which generates a new audio TSV manifest under `${PROCESSED_DATA_ROOT}` with updated path to the processed audio and -a new column for SNR. - -To do filtering by CER, follow the [Automatic Evaluation](../docs/ljspeech_example.md#automatic-evaluation) section to -run ASR model (add `--eval-target` to `get_eval_manifest` for evaluation on the reference audio; add `--err-unit char` -to `eval_asr` to compute CER instead of WER). The example-level CER is saved to -`${EVAL_OUTPUT_ROOT}/uer_cer.${SPLIT}.tsv`. - -Then, extract log-Mel spectrograms, generate feature manifest and create data configuration YAML with -```bash -python -m examples.speech_synthesis.preprocessing.get_feature_manifest \ - --audio-manifest-root ${PROCESSED_DATA_ROOT} \ - --output-root ${FEATURE_MANIFEST_ROOT} \ - --ipa-vocab --use-g2p \ - --snr-threshold 15 \ - --cer-threshold 0.1 --cer-tsv-path ${EVAL_OUTPUT_ROOT}/uer_cer.${SPLIT}.tsv -``` -where we use phoneme inputs (`--ipa-vocab --use-g2p`) as example. For sample filtering, we set the SNR and CER threshold -to 15 and 10%, respectively. - -## Training -(Please refer to [the LJSpeech example](../docs/ljspeech_example.md#transformer).) - -## Inference -(Please refer to [the LJSpeech example](../docs/ljspeech_example.md#inference).) - -## Automatic Evaluation -(Please refer to [the LJSpeech example](../docs/ljspeech_example.md#automatic-evaluation).) - -## Results - -| --arch | Params | Test MCD | Model | -|---|---|---|---| -| tts_transformer | 54M | 3.4 | [Download](https://dl.fbaipublicfiles.com/fairseq/s2/vctk_transformer_phn.tar) | - -[[Back]](..) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/evaluation/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/evaluation/__init__.py deleted file mode 100644 index 6264236915a7269a4d920ee8213004374dd86a9a..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/evaluation/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/evaluation/eval_asr.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/evaluation/eval_asr.py deleted file mode 100644 index 240e6ad723075300825164095193b73d5828361b..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/evaluation/eval_asr.py +++ /dev/null @@ -1,128 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import re -import shutil -import subprocess -from pathlib import Path - -import editdistance -import soundfile as sf -from examples.speech_to_text.data_utils import load_tsv_to_dicts - - -def preprocess_text(text): - text = "|".join(re.sub(r"[^A-Z' ]", " ", text.upper()).split()) - text = " ".join(text) - return text - - -def prepare_w2v_data( - dict_dir, sample_rate, label, audio_paths, texts, split, data_dir -): - data_dir.mkdir(parents=True, exist_ok=True) - shutil.copyfile( - dict_dir / f"dict.{label}.txt", - data_dir / f"dict.{label}.txt" - ) - with open(data_dir / f"{split}.tsv", "w") as f: - f.write("/\n") - for audio_path in audio_paths: - wav, sr = sf.read(audio_path) - assert sr == sample_rate, f"{sr} != sample_rate" - nsample = len(wav) - f.write(f"{audio_path}\t{nsample}\n") - with open(data_dir / f"{split}.{label}", "w") as f: - for text in texts: - text = preprocess_text(text) - f.write(f"{text}\n") - - -def run_asr(asr_dir, split, w2v_ckpt, w2v_label, res_dir): - """ - results will be saved at - {res_dir}/{ref,hypo}.word-{w2v_ckpt.filename}-{split}.txt - """ - cmd = ["python", "-m", "examples.speech_recognition.infer"] - cmd += [str(asr_dir.resolve())] - cmd += ["--task", "audio_finetuning", "--nbest", "1", "--quiet"] - cmd += ["--w2l-decoder", "viterbi", "--criterion", "ctc"] - cmd += ["--post-process", "letter", "--max-tokens", "4000000"] - cmd += ["--path", str(w2v_ckpt.resolve()), "--labels", w2v_label] - cmd += ["--gen-subset", split, "--results-path", str(res_dir.resolve())] - - print(f"running cmd:\n{' '.join(cmd)}") - subprocess.run(cmd, check=True) - - -def compute_error_rate(hyp_wrd_path, ref_wrd_path, unit="word"): - """each line is " (None-)" """ - tokenize_line = { - "word": lambda x: re.sub(r" \(.*\)$", "", x.rstrip()).split(), - "char": lambda x: list(re.sub(r" \(.*\)$", "", x.rstrip())) - }.get(unit) - if tokenize_line is None: - raise ValueError(f"{unit} not supported") - - inds = [int(re.sub(r"\D*(\d*)\D*", r"\1", line)) - for line in open(hyp_wrd_path)] - hyps = [tokenize_line(line) for line in open(hyp_wrd_path)] - refs = [tokenize_line(line) for line in open(ref_wrd_path)] - assert(len(hyps) == len(refs)) - err_rates = [ - editdistance.eval(hyp, ref) / len(ref) for hyp, ref in zip(hyps, refs) - ] - ind_to_err_rates = {i: e for i, e in zip(inds, err_rates)} - return ind_to_err_rates - - -def main(args): - samples = load_tsv_to_dicts(args.raw_manifest) - ids = [ - sample[args.id_header] if args.id_header else "" for sample in samples - ] - audio_paths = [sample[args.audio_header] for sample in samples] - texts = [sample[args.text_header] for sample in samples] - - prepare_w2v_data( - args.w2v_dict_dir, - args.w2v_sample_rate, - args.w2v_label, - audio_paths, - texts, - args.split, - args.asr_dir - ) - run_asr(args.asr_dir, args.split, args.w2v_ckpt, args.w2v_label, args.asr_dir) - ind_to_err_rates = compute_error_rate( - args.asr_dir / f"hypo.word-{args.w2v_ckpt.name}-{args.split}.txt", - args.asr_dir / f"ref.word-{args.w2v_ckpt.name}-{args.split}.txt", - args.err_unit, - ) - - uer_path = args.asr_dir / f"uer_{args.err_unit}.{args.split}.tsv" - with open(uer_path, "w") as f: - f.write("id\taudio\tuer\n") - for ind, (id_, audio_path) in enumerate(zip(ids, audio_paths)): - f.write(f"{id_}\t{audio_path}\t{ind_to_err_rates[ind]:.4f}\n") - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("--raw-manifest", required=True, type=Path) - parser.add_argument("--asr-dir", required=True, type=Path) - parser.add_argument("--id-header", default="id", type=str) - parser.add_argument("--audio-header", default="audio", type=str) - parser.add_argument("--text-header", default="src_text", type=str) - parser.add_argument("--split", default="raw", type=str) - parser.add_argument("--w2v-ckpt", required=True, type=Path) - parser.add_argument("--w2v-dict-dir", required=True, type=Path) - parser.add_argument("--w2v-sample-rate", default=16000, type=int) - parser.add_argument("--w2v-label", default="ltr", type=str) - parser.add_argument("--err-unit", default="word", type=str) - args = parser.parse_args() - - main(args) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/evaluation/eval_f0.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/evaluation/eval_f0.py deleted file mode 100644 index 6fc0d00e3f7b0469f750760bb3ad50d3112b66e8..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/evaluation/eval_f0.py +++ /dev/null @@ -1,267 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -""" -Signal processing-based evaluation using waveforms -""" -import os.path as op - -import numpy as np -import torchaudio -import tqdm -from examples.speech_synthesis.evaluation.eval_sp import load_eval_spec -from examples.speech_synthesis.utils import ( - f0_frame_error, - gross_pitch_error, - voicing_decision_error, -) -from tabulate import tabulate - - -def difference_function(x, n, tau_max): - """ - Compute difference function of data x. This solution is implemented directly - with Numpy fft. - - - :param x: audio data - :param n: length of data - :param tau_max: integration window size - :return: difference function - :rtype: list - """ - - x = np.array(x, np.float64) - w = x.size - tau_max = min(tau_max, w) - x_cumsum = np.concatenate((np.array([0.]), (x * x).cumsum())) - size = w + tau_max - p2 = (size // 32).bit_length() - nice_numbers = (16, 18, 20, 24, 25, 27, 30, 32) - size_pad = min(x * 2 ** p2 for x in nice_numbers if x * 2 ** p2 >= size) - fc = np.fft.rfft(x, size_pad) - conv = np.fft.irfft(fc * fc.conjugate())[:tau_max] - return x_cumsum[w:w - tau_max:-1] + x_cumsum[w] - x_cumsum[:tau_max] - \ - 2 * conv - - -def cumulative_mean_normalized_difference_function(df, n): - """ - Compute cumulative mean normalized difference function (CMND). - - :param df: Difference function - :param n: length of data - :return: cumulative mean normalized difference function - :rtype: list - """ - - # scipy method - cmn_df = df[1:] * range(1, n) / np.cumsum(df[1:]).astype(float) - return np.insert(cmn_df, 0, 1) - - -def get_pitch(cmdf, tau_min, tau_max, harmo_th=0.1): - """ - Return fundamental period of a frame based on CMND function. - - :param cmdf: Cumulative Mean Normalized Difference function - :param tau_min: minimum period for speech - :param tau_max: maximum period for speech - :param harmo_th: harmonicity threshold to determine if it is necessary to - compute pitch frequency - :return: fundamental period if there is values under threshold, 0 otherwise - :rtype: float - """ - tau = tau_min - while tau < tau_max: - if cmdf[tau] < harmo_th: - while tau + 1 < tau_max and cmdf[tau + 1] < cmdf[tau]: - tau += 1 - return tau - tau += 1 - - return 0 # if unvoiced - - -def compute_yin(sig, sr, w_len=512, w_step=256, f0_min=100, f0_max=500, - harmo_thresh=0.1): - """ - - Compute the Yin Algorithm. Return fundamental frequency and harmonic rate. - - https://github.com/NVIDIA/mellotron adaption of - https://github.com/patriceguyot/Yin - - :param sig: Audio signal (list of float) - :param sr: sampling rate (int) - :param w_len: size of the analysis window (samples) - :param w_step: size of the lag between two consecutives windows (samples) - :param f0_min: Minimum fundamental frequency that can be detected (hertz) - :param f0_max: Maximum fundamental frequency that can be detected (hertz) - :param harmo_thresh: Threshold of detection. The yalgorithmù return the - first minimum of the CMND function below this threshold. - - :returns: - - * pitches: list of fundamental frequencies, - * harmonic_rates: list of harmonic rate values for each fundamental - frequency value (= confidence value) - * argmins: minimums of the Cumulative Mean Normalized DifferenceFunction - * times: list of time of each estimation - :rtype: tuple - """ - - tau_min = int(sr / f0_max) - tau_max = int(sr / f0_min) - - # time values for each analysis window - time_scale = range(0, len(sig) - w_len, w_step) - times = [t/float(sr) for t in time_scale] - frames = [sig[t:t + w_len] for t in time_scale] - - pitches = [0.0] * len(time_scale) - harmonic_rates = [0.0] * len(time_scale) - argmins = [0.0] * len(time_scale) - - for i, frame in enumerate(frames): - # Compute YIN - df = difference_function(frame, w_len, tau_max) - cm_df = cumulative_mean_normalized_difference_function(df, tau_max) - p = get_pitch(cm_df, tau_min, tau_max, harmo_thresh) - - # Get results - if np.argmin(cm_df) > tau_min: - argmins[i] = float(sr / np.argmin(cm_df)) - if p != 0: # A pitch was found - pitches[i] = float(sr / p) - harmonic_rates[i] = cm_df[p] - else: # No pitch, but we compute a value of the harmonic rate - harmonic_rates[i] = min(cm_df) - - return pitches, harmonic_rates, argmins, times - - -def extract_f0(samples): - f0_samples = [] - for sample in tqdm.tqdm(samples): - if not op.isfile(sample["ref"]) or not op.isfile(sample["syn"]): - f0_samples.append(None) - continue - - # assume single channel - yref, sr = torchaudio.load(sample["ref"]) - ysyn, _sr = torchaudio.load(sample["syn"]) - yref, ysyn = yref[0], ysyn[0] - assert sr == _sr, f"{sr} != {_sr}" - - yref_f0 = compute_yin(yref, sr) - ysyn_f0 = compute_yin(ysyn, sr) - - f0_samples += [ - { - "ref": yref_f0, - "syn": ysyn_f0 - } - ] - - return f0_samples - - -def eval_f0_error(samples, distortion_fn): - results = [] - for sample in tqdm.tqdm(samples): - if sample is None: - results.append(None) - continue - # assume single channel - yref_f, _, _, yref_t = sample["ref"] - ysyn_f, _, _, ysyn_t = sample["syn"] - - yref_f = np.array(yref_f) - yref_t = np.array(yref_t) - ysyn_f = np.array(ysyn_f) - ysyn_t = np.array(ysyn_t) - - distortion = distortion_fn(yref_t, yref_f, ysyn_t, ysyn_f) - results.append((distortion.item(), - len(yref_f), - len(ysyn_f) - )) - return results - - -def eval_gross_pitch_error(samples): - return eval_f0_error(samples, gross_pitch_error) - - -def eval_voicing_decision_error(samples): - return eval_f0_error(samples, voicing_decision_error) - - -def eval_f0_frame_error(samples): - return eval_f0_error(samples, f0_frame_error) - - -def print_results(results, show_bin): - results = np.array(list(filter(lambda x: x is not None, results))) - - np.set_printoptions(precision=3) - - def _print_result(results): - res = { - "nutt": len(results), - "error": results[:, 0].mean(), - "std": results[:, 0].std(), - "dur_ref": int(results[:, 1].sum()), - "dur_syn": int(results[:, 2].sum()), - } - print(tabulate([res.values()], res.keys(), floatfmt=".4f")) - - print(">>>> ALL") - _print_result(results) - - if show_bin: - edges = [0, 200, 400, 600, 800, 1000, 2000, 4000] - for i in range(1, len(edges)): - mask = np.logical_and(results[:, 1] >= edges[i-1], - results[:, 1] < edges[i]) - if not mask.any(): - continue - bin_results = results[mask] - print(f">>>> ({edges[i-1]}, {edges[i]})") - _print_result(bin_results) - - -def main(eval_f0, gpe, vde, ffe, show_bin): - samples = load_eval_spec(eval_f0) - if gpe or vde or ffe: - f0_samples = extract_f0(samples) - - if gpe: - print("===== Evaluate Gross Pitch Error =====") - results = eval_gross_pitch_error(f0_samples) - print_results(results, show_bin) - if vde: - print("===== Evaluate Voicing Decision Error =====") - results = eval_voicing_decision_error(f0_samples) - print_results(results, show_bin) - if ffe: - print("===== Evaluate F0 Frame Error =====") - results = eval_f0_frame_error(f0_samples) - print_results(results, show_bin) - - -if __name__ == "__main__": - import argparse - - parser = argparse.ArgumentParser() - parser.add_argument("eval_f0") - parser.add_argument("--gpe", action="store_true") - parser.add_argument("--vde", action="store_true") - parser.add_argument("--ffe", action="store_true") - parser.add_argument("--show-bin", action="store_true") - args = parser.parse_args() - - main(args.eval_f0, args.gpe, args.vde, args.ffe, args.show_bin) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/evaluation/eval_sp.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/evaluation/eval_sp.py deleted file mode 100644 index 9bf58fcc0a2838fd592824d41392e1d2a464ff92..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/evaluation/eval_sp.py +++ /dev/null @@ -1,130 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - - -""" -Signal processing-based evaluation using waveforms -""" - -import csv -import os.path as op - -import numpy as np -import torch -import torchaudio -import tqdm -from examples.speech_synthesis.utils import batch_mel_spectral_distortion -from fairseq.tasks.text_to_speech import batch_mel_cepstral_distortion -from tabulate import tabulate - - -def load_eval_spec(path): - with open(path) as f: - reader = csv.DictReader(f, delimiter='\t') - samples = list(reader) - return samples - - -def eval_distortion(samples, distortion_fn, device="cuda"): - nmiss = 0 - results = [] - for sample in tqdm.tqdm(samples): - if not op.isfile(sample["ref"]) or not op.isfile(sample["syn"]): - nmiss += 1 - results.append(None) - continue - # assume single channel - yref, sr = torchaudio.load(sample["ref"]) - ysyn, _sr = torchaudio.load(sample["syn"]) - yref, ysyn = yref[0].to(device), ysyn[0].to(device) - assert sr == _sr, f"{sr} != {_sr}" - - distortion, extra = distortion_fn([yref], [ysyn], sr, None)[0] - _, _, _, _, _, pathmap = extra - nins = torch.sum(pathmap.sum(dim=1) - 1) # extra frames in syn - ndel = torch.sum(pathmap.sum(dim=0) - 1) # missing frames from syn - results.append( - (distortion.item(), # path distortion - pathmap.size(0), # yref num frames - pathmap.size(1), # ysyn num frames - pathmap.sum().item(), # path length - nins.item(), # insertion - ndel.item(), # deletion - ) - ) - return results - - -def eval_mel_cepstral_distortion(samples, device="cuda"): - return eval_distortion(samples, batch_mel_cepstral_distortion, device) - - -def eval_mel_spectral_distortion(samples, device="cuda"): - return eval_distortion(samples, batch_mel_spectral_distortion, device) - - -def print_results(results, show_bin): - results = np.array(list(filter(lambda x: x is not None, results))) - - np.set_printoptions(precision=3) - - def _print_result(results): - dist, dur_ref, dur_syn, dur_ali, nins, ndel = results.sum(axis=0) - res = { - "nutt": len(results), - "dist": dist, - "dur_ref": int(dur_ref), - "dur_syn": int(dur_syn), - "dur_ali": int(dur_ali), - "dist_per_ref_frm": dist/dur_ref, - "dist_per_syn_frm": dist/dur_syn, - "dist_per_ali_frm": dist/dur_ali, - "ins": nins/dur_ref, - "del": ndel/dur_ref, - } - print(tabulate( - [res.values()], - res.keys(), - floatfmt=".4f" - )) - - print(">>>> ALL") - _print_result(results) - - if show_bin: - edges = [0, 200, 400, 600, 800, 1000, 2000, 4000] - for i in range(1, len(edges)): - mask = np.logical_and(results[:, 1] >= edges[i-1], - results[:, 1] < edges[i]) - if not mask.any(): - continue - bin_results = results[mask] - print(f">>>> ({edges[i-1]}, {edges[i]})") - _print_result(bin_results) - - -def main(eval_spec, mcd, msd, show_bin): - samples = load_eval_spec(eval_spec) - device = "cpu" - if mcd: - print("===== Evaluate Mean Cepstral Distortion =====") - results = eval_mel_cepstral_distortion(samples, device) - print_results(results, show_bin) - if msd: - print("===== Evaluate Mean Spectral Distortion =====") - results = eval_mel_spectral_distortion(samples, device) - print_results(results, show_bin) - - -if __name__ == "__main__": - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("eval_spec") - parser.add_argument("--mcd", action="store_true") - parser.add_argument("--msd", action="store_true") - parser.add_argument("--show-bin", action="store_true") - args = parser.parse_args() - - main(args.eval_spec, args.mcd, args.msd, args.show_bin) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/evaluation/get_eval_manifest.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/evaluation/get_eval_manifest.py deleted file mode 100644 index 44b3685bb221a5bd4522668465902fb1d2eb40ec..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/evaluation/get_eval_manifest.py +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - - -import csv -from pathlib import Path - - -def main(args): - """ - `uid syn ref text` - """ - in_root = Path(args.generation_root).resolve() - ext = args.audio_format - with open(args.audio_manifest) as f, open(args.output_path, "w") as f_out: - reader = csv.DictReader( - f, delimiter="\t", quotechar=None, doublequote=False, - lineterminator="\n", quoting=csv.QUOTE_NONE - ) - header = ["id", "syn", "ref", "text", "speaker"] - f_out.write("\t".join(header) + "\n") - for row in reader: - dir_name = f"{ext}_{args.sample_rate}hz_{args.vocoder}" - id_ = row["id"] - syn = (in_root / dir_name / f"{id_}.{ext}").as_posix() - ref = row["audio"] - if args.use_resynthesized_target: - ref = (in_root / f"{dir_name}_tgt" / f"{id_}.{ext}").as_posix() - if args.eval_target: - syn = row["audio"] - sample = [id_, syn, ref, row["tgt_text"], row["speaker"]] - f_out.write("\t".join(sample) + "\n") - print(f"wrote evaluation file to {args.output_path}") - - -if __name__ == "__main__": - import argparse - parser = argparse.ArgumentParser() - parser.add_argument( - "--generation-root", help="output directory for generate_waveform.py" - ) - parser.add_argument( - "--audio-manifest", - help="used to determine the original utterance ID and text" - ) - parser.add_argument( - "--output-path", help="path to output evaluation spec file" - ) - parser.add_argument( - "--use-resynthesized-target", action="store_true", - help="use resynthesized reference instead of the original audio" - ) - parser.add_argument( - "--eval-target", action="store_true", - help="evaluate reference instead of model prediction" - ) - parser.add_argument("--vocoder", type=str, default="griffin_lim") - parser.add_argument("--sample-rate", type=int, default=22_050) - parser.add_argument("--audio-format", type=str, default="wav") - args = parser.parse_args() - - main(args) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/generate_waveform.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/generate_waveform.py deleted file mode 100644 index 9320508cdabe5c6d7eb8f91f8656e4ecda9ea9b9..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/generate_waveform.py +++ /dev/null @@ -1,191 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import ast -import logging -import sys -from pathlib import Path - -import matplotlib.pyplot as plt -import numpy as np -import soundfile as sf -import torch -import torchaudio -from fairseq import checkpoint_utils, options, tasks, utils -from fairseq.data.audio.text_to_speech_dataset import TextToSpeechDataset -from fairseq.logging import progress_bar -from fairseq.tasks.text_to_speech import plot_tts_output - -logging.basicConfig() -logging.root.setLevel(logging.INFO) -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - - -def make_parser(): - parser = options.get_speech_generation_parser() - parser.add_argument("--dump-features", action="store_true") - parser.add_argument("--dump-waveforms", action="store_true") - parser.add_argument("--dump-attentions", action="store_true") - parser.add_argument("--dump-eos-probs", action="store_true") - parser.add_argument("--dump-plots", action="store_true") - parser.add_argument("--dump-target", action="store_true") - parser.add_argument("--output-sample-rate", default=22050, type=int) - parser.add_argument("--teacher-forcing", action="store_true") - parser.add_argument( - "--audio-format", type=str, default="wav", choices=["wav", "flac"] - ) - return parser - - -def postprocess_results( - dataset: TextToSpeechDataset, sample, hypos, resample_fn, dump_target -): - def to_np(x): - return None if x is None else x.detach().cpu().numpy() - - sample_ids = [dataset.ids[i] for i in sample["id"].tolist()] - texts = sample["src_texts"] if "src_texts" in sample else [""] * len(hypos) - attns = [to_np(hypo["attn"]) for hypo in hypos] - eos_probs = [to_np(hypo.get("eos_prob", None)) for hypo in hypos] - feat_preds = [to_np(hypo["feature"]) for hypo in hypos] - wave_preds = [to_np(resample_fn(h["waveform"])) for h in hypos] - if dump_target: - feat_targs = [to_np(hypo["targ_feature"]) for hypo in hypos] - wave_targs = [to_np(resample_fn(h["targ_waveform"])) for h in hypos] - else: - feat_targs = [None for _ in hypos] - wave_targs = [None for _ in hypos] - - return zip(sample_ids, texts, attns, eos_probs, feat_preds, wave_preds, - feat_targs, wave_targs) - - -def dump_result( - is_na_model, - args, - vocoder, - sample_id, - text, - attn, - eos_prob, - feat_pred, - wave_pred, - feat_targ, - wave_targ, -): - sample_rate = args.output_sample_rate - out_root = Path(args.results_path) - if args.dump_features: - feat_dir = out_root / "feat" - feat_dir.mkdir(exist_ok=True, parents=True) - np.save(feat_dir / f"{sample_id}.npy", feat_pred) - if args.dump_target: - feat_tgt_dir = out_root / "feat_tgt" - feat_tgt_dir.mkdir(exist_ok=True, parents=True) - np.save(feat_tgt_dir / f"{sample_id}.npy", feat_targ) - if args.dump_attentions: - attn_dir = out_root / "attn" - attn_dir.mkdir(exist_ok=True, parents=True) - np.save(attn_dir / f"{sample_id}.npy", attn.numpy()) - if args.dump_eos_probs and not is_na_model: - eos_dir = out_root / "eos" - eos_dir.mkdir(exist_ok=True, parents=True) - np.save(eos_dir / f"{sample_id}.npy", eos_prob) - - if args.dump_plots: - images = [feat_pred.T] if is_na_model else [feat_pred.T, attn] - names = ["output"] if is_na_model else ["output", "alignment"] - if feat_targ is not None: - images = [feat_targ.T] + images - names = [f"target (idx={sample_id})"] + names - if is_na_model: - plot_tts_output(images, names, attn, "alignment", suptitle=text) - else: - plot_tts_output(images, names, eos_prob, "eos prob", suptitle=text) - plot_dir = out_root / "plot" - plot_dir.mkdir(exist_ok=True, parents=True) - plt.savefig(plot_dir / f"{sample_id}.png") - plt.close() - - if args.dump_waveforms: - ext = args.audio_format - if wave_pred is not None: - wav_dir = out_root / f"{ext}_{sample_rate}hz_{vocoder}" - wav_dir.mkdir(exist_ok=True, parents=True) - sf.write(wav_dir / f"{sample_id}.{ext}", wave_pred, sample_rate) - if args.dump_target and wave_targ is not None: - wav_tgt_dir = out_root / f"{ext}_{sample_rate}hz_{vocoder}_tgt" - wav_tgt_dir.mkdir(exist_ok=True, parents=True) - sf.write(wav_tgt_dir / f"{sample_id}.{ext}", wave_targ, sample_rate) - - -def main(args): - assert(args.dump_features or args.dump_waveforms or args.dump_attentions - or args.dump_eos_probs or args.dump_plots) - if args.max_tokens is None and args.batch_size is None: - args.max_tokens = 8000 - logger.info(args) - - use_cuda = torch.cuda.is_available() and not args.cpu - task = tasks.setup_task(args) - models, saved_cfg, task = checkpoint_utils.load_model_ensemble_and_task( - [args.path], - task=task, - arg_overrides=ast.literal_eval(args.model_overrides), - ) - model = models[0].cuda() if use_cuda else models[0] - # use the original n_frames_per_step - task.args.n_frames_per_step = saved_cfg.task.n_frames_per_step - task.load_dataset(args.gen_subset, task_cfg=saved_cfg.task) - - data_cfg = task.data_cfg - sample_rate = data_cfg.config.get("features", {}).get("sample_rate", 22050) - resample_fn = { - False: lambda x: x, - True: lambda x: torchaudio.sox_effects.apply_effects_tensor( - x.detach().cpu().unsqueeze(0), sample_rate, - [['rate', str(args.output_sample_rate)]] - )[0].squeeze(0) - }.get(args.output_sample_rate != sample_rate) - if args.output_sample_rate != sample_rate: - logger.info(f"resampling to {args.output_sample_rate}Hz") - - generator = task.build_generator([model], args) - itr = task.get_batch_iterator( - dataset=task.dataset(args.gen_subset), - max_tokens=args.max_tokens, - max_sentences=args.batch_size, - max_positions=(sys.maxsize, sys.maxsize), - ignore_invalid_inputs=args.skip_invalid_size_inputs_valid_test, - required_batch_size_multiple=args.required_batch_size_multiple, - num_shards=args.num_shards, - shard_id=args.shard_id, - num_workers=args.num_workers, - data_buffer_size=args.data_buffer_size, - ).next_epoch_itr(shuffle=False) - - Path(args.results_path).mkdir(exist_ok=True, parents=True) - is_na_model = getattr(model, "NON_AUTOREGRESSIVE", False) - dataset = task.dataset(args.gen_subset) - vocoder = task.args.vocoder - with progress_bar.build_progress_bar(args, itr) as t: - for sample in t: - sample = utils.move_to_cuda(sample) if use_cuda else sample - hypos = generator.generate(model, sample, has_targ=args.dump_target) - for result in postprocess_results( - dataset, sample, hypos, resample_fn, args.dump_target - ): - dump_result(is_na_model, args, vocoder, *result) - - -def cli_main(): - parser = make_parser() - args = options.parse_args_and_arch(parser) - main(args) - - -if __name__ == "__main__": - cli_main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/preprocessing/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/preprocessing/__init__.py deleted file mode 100644 index 6264236915a7269a4d920ee8213004374dd86a9a..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/preprocessing/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/preprocessing/denoise_and_vad_audio.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/preprocessing/denoise_and_vad_audio.py deleted file mode 100644 index 9b6399aac3331d73416a020dd5784bac9d951aea..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/preprocessing/denoise_and_vad_audio.py +++ /dev/null @@ -1,208 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import csv -import logging -import os -import tempfile -from collections import defaultdict -from pathlib import Path - -import torchaudio - -try: - import webrtcvad -except ImportError: - raise ImportError("Please install py-webrtcvad: pip install webrtcvad") -import examples.speech_synthesis.preprocessing.denoiser.utils as utils -import pandas as pd -from examples.speech_synthesis.preprocessing.denoiser.pretrained import master64 -from examples.speech_synthesis.preprocessing.vad import ( - FS_MS, - SCALE, - THRESHOLD, - frame_generator, - read_wave, - vad_collector, - write_wave, -) -from examples.speech_to_text.data_utils import save_df_to_tsv -from tqdm import tqdm - -log = logging.getLogger(__name__) - -PATHS = ["after_denoise", "after_vad"] -MIN_T = 0.05 - - -def generate_tmp_filename(extension="txt"): - return tempfile._get_default_tempdir() + "/" + \ - next(tempfile._get_candidate_names()) + "." + extension - - -def convert_sr(inpath, sr, output_path=None): - if not output_path: - output_path = generate_tmp_filename("wav") - cmd = f"sox {inpath} -r {sr} {output_path}" - os.system(cmd) - return output_path - - -def apply_vad(vad, inpath): - audio, sample_rate = read_wave(inpath) - frames = frame_generator(FS_MS, audio, sample_rate) - frames = list(frames) - segments = vad_collector(sample_rate, FS_MS, 300, vad, frames) - merge_segments = list() - timestamp_start = 0.0 - timestamp_end = 0.0 - # removing start, end, and long sequences of sils - for i, segment in enumerate(segments): - merge_segments.append(segment[0]) - if i and timestamp_start: - sil_duration = segment[1] - timestamp_end - if sil_duration > THRESHOLD: - merge_segments.append(int(THRESHOLD / SCALE) * (b'\x00')) - else: - merge_segments.append(int((sil_duration / SCALE)) * (b'\x00')) - timestamp_start = segment[1] - timestamp_end = segment[2] - segment = b''.join(merge_segments) - return segment, sample_rate - - -def write(wav, filename, sr=16_000): - # Normalize audio if it prevents clipping - wav = wav / max(wav.abs().max().item(), 1) - torchaudio.save(filename, wav.cpu(), sr, encoding="PCM_S", - bits_per_sample=16) - - -def process(args): - # making sure we are requested either denoise or vad - if not args.denoise and not args.vad: - log.error("No denoise or vad is requested.") - return - - log.info("Creating out directories...") - if args.denoise: - out_denoise = Path(args.output_dir).absolute().joinpath(PATHS[0]) - out_denoise.mkdir(parents=True, exist_ok=True) - if args.vad: - out_vad = Path(args.output_dir).absolute().joinpath(PATHS[1]) - out_vad.mkdir(parents=True, exist_ok=True) - - log.info("Loading pre-trained speech enhancement model...") - model = master64().to(args.device) - - log.info("Building the VAD model...") - vad = webrtcvad.Vad(int(args.vad_agg_level)) - - # preparing the output dict - output_dict = defaultdict(list) - - log.info(f"Parsing input manifest: {args.audio_manifest}") - with open(args.audio_manifest, "r") as f: - manifest_dict = csv.DictReader(f, delimiter="\t") - for row in tqdm(manifest_dict): - filename = str(row["audio"]) - - final_output = filename - keep_sample = True - n_frames = row["n_frames"] - snr = -1 - if args.denoise: - output_path_denoise = out_denoise.joinpath(Path(filename).name) - # convert to 16khz in case we use a differet sr - tmp_path = convert_sr(final_output, 16000) - - # loading audio file and generating the enhanced version - out, sr = torchaudio.load(tmp_path) - out = out.to(args.device) - estimate = model(out) - estimate = (1 - args.dry_wet) * estimate + args.dry_wet * out - write(estimate[0], str(output_path_denoise), sr) - - snr = utils.cal_snr(out, estimate) - snr = snr.cpu().detach().numpy()[0][0] - final_output = str(output_path_denoise) - - if args.vad: - output_path_vad = out_vad.joinpath(Path(filename).name) - sr = torchaudio.info(final_output).sample_rate - if sr in [16000, 32000, 48000]: - tmp_path = final_output - elif sr < 16000: - tmp_path = convert_sr(final_output, 16000) - elif sr < 32000: - tmp_path = convert_sr(final_output, 32000) - else: - tmp_path = convert_sr(final_output, 48000) - # apply VAD - segment, sample_rate = apply_vad(vad, tmp_path) - if len(segment) < sample_rate * MIN_T: - keep_sample = False - print(( - f"WARNING: skip {filename} because it is too short " - f"after VAD ({len(segment) / sample_rate} < {MIN_T})" - )) - else: - if sample_rate != sr: - tmp_path = generate_tmp_filename("wav") - write_wave(tmp_path, segment, sample_rate) - convert_sr(tmp_path, sr, - output_path=str(output_path_vad)) - else: - write_wave(str(output_path_vad), segment, sample_rate) - final_output = str(output_path_vad) - segment, _ = torchaudio.load(final_output) - n_frames = segment.size(1) - - if keep_sample: - output_dict["id"].append(row["id"]) - output_dict["audio"].append(final_output) - output_dict["n_frames"].append(n_frames) - output_dict["tgt_text"].append(row["tgt_text"]) - output_dict["speaker"].append(row["speaker"]) - output_dict["src_text"].append(row["src_text"]) - output_dict["snr"].append(snr) - - out_tsv_path = Path(args.output_dir) / Path(args.audio_manifest).name - log.info(f"Saving manifest to {out_tsv_path.as_posix()}") - save_df_to_tsv(pd.DataFrame.from_dict(output_dict), out_tsv_path) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--audio-manifest", "-i", required=True, - type=str, help="path to the input manifest.") - parser.add_argument( - "--output-dir", "-o", required=True, type=str, - help="path to the output dir. it will contain files after denoising and" - " vad" - ) - parser.add_argument("--vad-agg-level", "-a", type=int, default=2, - help="the aggresive level of the vad [0-3].") - parser.add_argument( - "--dry-wet", "-dw", type=float, default=0.01, - help="the level of linear interpolation between noisy and enhanced " - "files." - ) - parser.add_argument( - "--device", "-d", type=str, default="cpu", - help="the device to be used for the speech enhancement model: " - "cpu | cuda." - ) - parser.add_argument("--denoise", action="store_true", - help="apply a denoising") - parser.add_argument("--vad", action="store_true", help="apply a VAD") - args = parser.parse_args() - - process(args) - - -if __name__ == "__main__": - main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/preprocessing/denoiser/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/preprocessing/denoiser/__init__.py deleted file mode 100644 index 6264236915a7269a4d920ee8213004374dd86a9a..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/preprocessing/denoiser/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/preprocessing/denoiser/demucs.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/preprocessing/denoiser/demucs.py deleted file mode 100644 index 3f70e73d6a37d32e05b6cf0e87f42e13c467cd52..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/preprocessing/denoiser/demucs.py +++ /dev/null @@ -1,473 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# All rights reserved. -# -# This source code is licensed under the license found in the -# LICENSE file in the root directory of this source tree. -# author: adefossez - -import math -import time - -import torch as th -from torch import nn -from torch.nn import functional as F - -from .resample import downsample2, upsample2 -from .utils import capture_init - - -class BLSTM(nn.Module): - def __init__(self, dim, layers=2, bi=True): - super().__init__() - klass = nn.LSTM - self.lstm = klass( - bidirectional=bi, num_layers=layers, hidden_size=dim, input_size=dim - ) - self.linear = None - if bi: - self.linear = nn.Linear(2 * dim, dim) - - def forward(self, x, hidden=None): - x, hidden = self.lstm(x, hidden) - if self.linear: - x = self.linear(x) - return x, hidden - - -def rescale_conv(conv, reference): - std = conv.weight.std().detach() - scale = (std / reference)**0.5 - conv.weight.data /= scale - if conv.bias is not None: - conv.bias.data /= scale - - -def rescale_module(module, reference): - for sub in module.modules(): - if isinstance(sub, (nn.Conv1d, nn.ConvTranspose1d)): - rescale_conv(sub, reference) - - -class Demucs(nn.Module): - """ - Demucs speech enhancement model. - Args: - - chin (int): number of input channels. - - chout (int): number of output channels. - - hidden (int): number of initial hidden channels. - - depth (int): number of layers. - - kernel_size (int): kernel size for each layer. - - stride (int): stride for each layer. - - causal (bool): if false, uses BiLSTM instead of LSTM. - - resample (int): amount of resampling to apply to the input/output. - Can be one of 1, 2 or 4. - - growth (float): number of channels is multiplied by this for every layer. - - max_hidden (int): maximum number of channels. Can be useful to - control the size/speed of the model. - - normalize (bool): if true, normalize the input. - - glu (bool): if true uses GLU instead of ReLU in 1x1 convolutions. - - rescale (float): controls custom weight initialization. - See https://arxiv.org/abs/1911.13254. - - floor (float): stability flooring when normalizing. - - """ - @capture_init - def __init__(self, - chin=1, - chout=1, - hidden=48, - depth=5, - kernel_size=8, - stride=4, - causal=True, - resample=4, - growth=2, - max_hidden=10_000, - normalize=True, - glu=True, - rescale=0.1, - floor=1e-3): - - super().__init__() - if resample not in [1, 2, 4]: - raise ValueError("Resample should be 1, 2 or 4.") - - self.chin = chin - self.chout = chout - self.hidden = hidden - self.depth = depth - self.kernel_size = kernel_size - self.stride = stride - self.causal = causal - self.floor = floor - self.resample = resample - self.normalize = normalize - - self.encoder = nn.ModuleList() - self.decoder = nn.ModuleList() - activation = nn.GLU(1) if glu else nn.ReLU() - ch_scale = 2 if glu else 1 - - for index in range(depth): - encode = [] - encode += [ - nn.Conv1d(chin, hidden, kernel_size, stride), - nn.ReLU(), - nn.Conv1d(hidden, hidden * ch_scale, 1), activation, - ] - self.encoder.append(nn.Sequential(*encode)) - - decode = [] - decode += [ - nn.Conv1d(hidden, ch_scale * hidden, 1), activation, - nn.ConvTranspose1d(hidden, chout, kernel_size, stride), - ] - if index > 0: - decode.append(nn.ReLU()) - self.decoder.insert(0, nn.Sequential(*decode)) - chout = hidden - chin = hidden - hidden = min(int(growth * hidden), max_hidden) - - self.lstm = BLSTM(chin, bi=not causal) - if rescale: - rescale_module(self, reference=rescale) - - def valid_length(self, length): - """ - Return the nearest valid length to use with the model so that - there is no time steps left over in a convolutions, e.g. for all - layers, size of the input - kernel_size % stride = 0. - - If the mixture has a valid length, the estimated sources - will have exactly the same length. - """ - length = math.ceil(length * self.resample) - for _ in range(self.depth): - length = math.ceil((length - self.kernel_size) / self.stride) + 1 - length = max(length, 1) - for _ in range(self.depth): - length = (length - 1) * self.stride + self.kernel_size - length = int(math.ceil(length / self.resample)) - return int(length) - - @property - def total_stride(self): - return self.stride ** self.depth // self.resample - - def forward(self, mix): - if mix.dim() == 2: - mix = mix.unsqueeze(1) - - if self.normalize: - mono = mix.mean(dim=1, keepdim=True) - std = mono.std(dim=-1, keepdim=True) - mix = mix / (self.floor + std) - else: - std = 1 - length = mix.shape[-1] - x = mix - x = F.pad(x, (0, self.valid_length(length) - length)) - if self.resample == 2: - x = upsample2(x) - elif self.resample == 4: - x = upsample2(x) - x = upsample2(x) - skips = [] - for encode in self.encoder: - x = encode(x) - skips.append(x) - x = x.permute(2, 0, 1) - x, _ = self.lstm(x) - x = x.permute(1, 2, 0) - for decode in self.decoder: - skip = skips.pop(-1) - x = x + skip[..., :x.shape[-1]] - x = decode(x) - if self.resample == 2: - x = downsample2(x) - elif self.resample == 4: - x = downsample2(x) - x = downsample2(x) - - x = x[..., :length] - return std * x - - -def fast_conv(conv, x): - """ - Faster convolution evaluation if either kernel size is 1 - or length of sequence is 1. - """ - batch, chin, length = x.shape - chout, chin, kernel = conv.weight.shape - assert batch == 1 - if kernel == 1: - x = x.view(chin, length) - out = th.addmm(conv.bias.view(-1, 1), - conv.weight.view(chout, chin), x) - elif length == kernel: - x = x.view(chin * kernel, 1) - out = th.addmm(conv.bias.view(-1, 1), - conv.weight.view(chout, chin * kernel), x) - else: - out = conv(x) - return out.view(batch, chout, -1) - - -class DemucsStreamer: - """ - Streaming implementation for Demucs. It supports being fed with any amount - of audio at a time. You will get back as much audio as possible at that - point. - - Args: - - demucs (Demucs): Demucs model. - - dry (float): amount of dry (e.g. input) signal to keep. 0 is maximum - noise removal, 1 just returns the input signal. Small values > 0 - allows to limit distortions. - - num_frames (int): number of frames to process at once. Higher values - will increase overall latency but improve the real time factor. - - resample_lookahead (int): extra lookahead used for the resampling. - - resample_buffer (int): size of the buffer of previous inputs/outputs - kept for resampling. - """ - def __init__(self, demucs, - dry=0, - num_frames=1, - resample_lookahead=64, - resample_buffer=256): - device = next(iter(demucs.parameters())).device - self.demucs = demucs - self.lstm_state = None - self.conv_state = None - self.dry = dry - self.resample_lookahead = resample_lookahead - resample_buffer = min(demucs.total_stride, resample_buffer) - self.resample_buffer = resample_buffer - self.frame_length = demucs.valid_length(1) + \ - demucs.total_stride * (num_frames - 1) - self.total_length = self.frame_length + self.resample_lookahead - self.stride = demucs.total_stride * num_frames - self.resample_in = th.zeros(demucs.chin, resample_buffer, device=device) - self.resample_out = th.zeros( - demucs.chin, resample_buffer, device=device - ) - - self.frames = 0 - self.total_time = 0 - self.variance = 0 - self.pending = th.zeros(demucs.chin, 0, device=device) - - bias = demucs.decoder[0][2].bias - weight = demucs.decoder[0][2].weight - chin, chout, kernel = weight.shape - self._bias = bias.view(-1, 1).repeat(1, kernel).view(-1, 1) - self._weight = weight.permute(1, 2, 0).contiguous() - - def reset_time_per_frame(self): - self.total_time = 0 - self.frames = 0 - - @property - def time_per_frame(self): - return self.total_time / self.frames - - def flush(self): - """ - Flush remaining audio by padding it with zero. Call this - when you have no more input and want to get back the last chunk of audio. - """ - pending_length = self.pending.shape[1] - padding = th.zeros( - self.demucs.chin, self.total_length, device=self.pending.device - ) - out = self.feed(padding) - return out[:, :pending_length] - - def feed(self, wav): - """ - Apply the model to mix using true real time evaluation. - Normalization is done online as is the resampling. - """ - begin = time.time() - demucs = self.demucs - resample_buffer = self.resample_buffer - stride = self.stride - resample = demucs.resample - - if wav.dim() != 2: - raise ValueError("input wav should be two dimensional.") - chin, _ = wav.shape - if chin != demucs.chin: - raise ValueError(f"Expected {demucs.chin} channels, got {chin}") - - self.pending = th.cat([self.pending, wav], dim=1) - outs = [] - while self.pending.shape[1] >= self.total_length: - self.frames += 1 - frame = self.pending[:, :self.total_length] - dry_signal = frame[:, :stride] - if demucs.normalize: - mono = frame.mean(0) - variance = (mono**2).mean() - self.variance = variance / self.frames + \ - (1 - 1 / self.frames) * self.variance - frame = frame / (demucs.floor + math.sqrt(self.variance)) - frame = th.cat([self.resample_in, frame], dim=-1) - self.resample_in[:] = frame[:, stride - resample_buffer:stride] - - if resample == 4: - frame = upsample2(upsample2(frame)) - elif resample == 2: - frame = upsample2(frame) - # remove pre sampling buffer - frame = frame[:, resample * resample_buffer:] - # remove extra samples after window - frame = frame[:, :resample * self.frame_length] - - out, extra = self._separate_frame(frame) - padded_out = th.cat([self.resample_out, out, extra], 1) - self.resample_out[:] = out[:, -resample_buffer:] - if resample == 4: - out = downsample2(downsample2(padded_out)) - elif resample == 2: - out = downsample2(padded_out) - else: - out = padded_out - - out = out[:, resample_buffer // resample:] - out = out[:, :stride] - - if demucs.normalize: - out *= math.sqrt(self.variance) - out = self.dry * dry_signal + (1 - self.dry) * out - outs.append(out) - self.pending = self.pending[:, stride:] - - self.total_time += time.time() - begin - if outs: - out = th.cat(outs, 1) - else: - out = th.zeros(chin, 0, device=wav.device) - return out - - def _separate_frame(self, frame): - demucs = self.demucs - skips = [] - next_state = [] - first = self.conv_state is None - stride = self.stride * demucs.resample - x = frame[None] - for idx, encode in enumerate(demucs.encoder): - stride //= demucs.stride - length = x.shape[2] - if idx == demucs.depth - 1: - # This is sligthly faster for the last conv - x = fast_conv(encode[0], x) - x = encode[1](x) - x = fast_conv(encode[2], x) - x = encode[3](x) - else: - if not first: - prev = self.conv_state.pop(0) - prev = prev[..., stride:] - tgt = (length - demucs.kernel_size) // demucs.stride + 1 - missing = tgt - prev.shape[-1] - offset = length - demucs.kernel_size - \ - demucs.stride * (missing - 1) - x = x[..., offset:] - x = encode[1](encode[0](x)) - x = fast_conv(encode[2], x) - x = encode[3](x) - if not first: - x = th.cat([prev, x], -1) - next_state.append(x) - skips.append(x) - - x = x.permute(2, 0, 1) - x, self.lstm_state = demucs.lstm(x, self.lstm_state) - x = x.permute(1, 2, 0) - # In the following, x contains only correct samples, i.e. the one - # for which each time position is covered by two window of the upper - # layer. extra contains extra samples to the right, and is used only as - # a better padding for the online resampling. - extra = None - for idx, decode in enumerate(demucs.decoder): - skip = skips.pop(-1) - x += skip[..., :x.shape[-1]] - x = fast_conv(decode[0], x) - x = decode[1](x) - - if extra is not None: - skip = skip[..., x.shape[-1]:] - extra += skip[..., :extra.shape[-1]] - extra = decode[2](decode[1](decode[0](extra))) - x = decode[2](x) - next_state.append( - x[..., -demucs.stride:] - decode[2].bias.view(-1, 1) - ) - if extra is None: - extra = x[..., -demucs.stride:] - else: - extra[..., :demucs.stride] += next_state[-1] - x = x[..., :-demucs.stride] - - if not first: - prev = self.conv_state.pop(0) - x[..., :demucs.stride] += prev - if idx != demucs.depth - 1: - x = decode[3](x) - extra = decode[3](extra) - self.conv_state = next_state - return x[0], extra[0] - - -def test(): - import argparse - parser = argparse.ArgumentParser( - "denoiser.demucs", - description="Benchmark the streaming Demucs implementation, as well as " - "checking the delta with the offline implementation.") - parser.add_argument("--depth", default=5, type=int) - parser.add_argument("--resample", default=4, type=int) - parser.add_argument("--hidden", default=48, type=int) - parser.add_argument("--sample_rate", default=16000, type=float) - parser.add_argument("--device", default="cpu") - parser.add_argument("-t", "--num_threads", type=int) - parser.add_argument("-f", "--num_frames", type=int, default=1) - args = parser.parse_args() - if args.num_threads: - th.set_num_threads(args.num_threads) - sr = args.sample_rate - sr_ms = sr / 1000 - demucs = Demucs( - depth=args.depth, hidden=args.hidden, resample=args.resample - ).to(args.device) - x = th.randn(1, int(sr * 4)).to(args.device) - out = demucs(x[None])[0] - streamer = DemucsStreamer(demucs, num_frames=args.num_frames) - out_rt = [] - frame_size = streamer.total_length - with th.no_grad(): - while x.shape[1] > 0: - out_rt.append(streamer.feed(x[:, :frame_size])) - x = x[:, frame_size:] - frame_size = streamer.demucs.total_stride - out_rt.append(streamer.flush()) - out_rt = th.cat(out_rt, 1) - model_size = sum(p.numel() for p in demucs.parameters()) * 4 / 2**20 - initial_lag = streamer.total_length / sr_ms - tpf = 1000 * streamer.time_per_frame - print(f"model size: {model_size:.1f}MB, ", end='') - print(f"delta batch/streaming: {th.norm(out - out_rt) / th.norm(out):.2%}") - print(f"initial lag: {initial_lag:.1f}ms, ", end='') - print(f"stride: {streamer.stride * args.num_frames / sr_ms:.1f}ms") - print(f"time per frame: {tpf:.1f}ms, ", end='') - rtf = (1000 * streamer.time_per_frame) / (streamer.stride / sr_ms) - print(f"RTF: {rtf:.2f}") - print(f"Total lag with computation: {initial_lag + tpf:.1f}ms") - - -if __name__ == "__main__": - test() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/preprocessing/denoiser/pretrained.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/preprocessing/denoiser/pretrained.py deleted file mode 100644 index 2fa846075b6872cdcc0baebca0b9acbb9ffcd287..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/preprocessing/denoiser/pretrained.py +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# All rights reserved. -# -# This source code is licensed under the license found in the -# LICENSE file in the root directory of this source tree. -# author: adefossez - -import logging - -import torch.hub - -from .demucs import Demucs -from .utils import deserialize_model - -logger = logging.getLogger(__name__) -ROOT = "https://dl.fbaipublicfiles.com/adiyoss/denoiser/" -DNS_48_URL = ROOT + "dns48-11decc9d8e3f0998.th" -DNS_64_URL = ROOT + "dns64-a7761ff99a7d5bb6.th" -MASTER_64_URL = ROOT + "master64-8a5dfb4bb92753dd.th" - - -def _demucs(pretrained, url, **kwargs): - model = Demucs(**kwargs) - if pretrained: - state_dict = torch.hub.load_state_dict_from_url(url, map_location='cpu') - model.load_state_dict(state_dict) - return model - - -def dns48(pretrained=True): - return _demucs(pretrained, DNS_48_URL, hidden=48) - - -def dns64(pretrained=True): - return _demucs(pretrained, DNS_64_URL, hidden=64) - - -def master64(pretrained=True): - return _demucs(pretrained, MASTER_64_URL, hidden=64) - - -def add_model_flags(parser): - group = parser.add_mutually_exclusive_group(required=False) - group.add_argument( - "-m", "--model_path", help="Path to local trained model." - ) - group.add_argument( - "--dns48", action="store_true", - help="Use pre-trained real time H=48 model trained on DNS." - ) - group.add_argument( - "--dns64", action="store_true", - help="Use pre-trained real time H=64 model trained on DNS." - ) - group.add_argument( - "--master64", action="store_true", - help="Use pre-trained real time H=64 model trained on DNS and Valentini." - ) - - -def get_model(args): - """ - Load local model package or torchhub pre-trained model. - """ - if args.model_path: - logger.info("Loading model from %s", args.model_path) - pkg = torch.load(args.model_path) - model = deserialize_model(pkg) - elif args.dns64: - logger.info("Loading pre-trained real time H=64 model trained on DNS.") - model = dns64() - elif args.master64: - logger.info( - "Loading pre-trained real time H=64 model trained on DNS and Valentini." - ) - model = master64() - else: - logger.info("Loading pre-trained real time H=48 model trained on DNS.") - model = dns48() - logger.debug(model) - return model diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/preprocessing/denoiser/resample.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/preprocessing/denoiser/resample.py deleted file mode 100644 index 1222addc424d4f898d602009e4032907241aadfe..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/preprocessing/denoiser/resample.py +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# All rights reserved. -# -# This source code is licensed under the license found in the -# LICENSE file in the root directory of this source tree. -# author: adefossez - -import math - -import torch as th -from torch.nn import functional as F - - -def sinc(t): - """sinc. - - :param t: the input tensor - """ - return th.where(t == 0, th.tensor(1., device=t.device, dtype=t.dtype), - th.sin(t) / t) - - -def kernel_upsample2(zeros=56): - """kernel_upsample2. - - """ - win = th.hann_window(4 * zeros + 1, periodic=False) - winodd = win[1::2] - t = th.linspace(-zeros + 0.5, zeros - 0.5, 2 * zeros) - t *= math.pi - kernel = (sinc(t) * winodd).view(1, 1, -1) - return kernel - - -def upsample2(x, zeros=56): - """ - Upsampling the input by 2 using sinc interpolation. - Smith, Julius, and Phil Gossett. "A flexible sampling-rate conversion method." - ICASSP'84. IEEE International Conference on Acoustics, Speech, and Signal Processing. - Vol. 9. IEEE, 1984. - """ - *other, time = x.shape - kernel = kernel_upsample2(zeros).to(x) - out = F.conv1d(x.view(-1, 1, time), kernel, padding=zeros)[..., 1:].view( - *other, time - ) - y = th.stack([x, out], dim=-1) - return y.view(*other, -1) - - -def kernel_downsample2(zeros=56): - """kernel_downsample2. - - """ - win = th.hann_window(4 * zeros + 1, periodic=False) - winodd = win[1::2] - t = th.linspace(-zeros + 0.5, zeros - 0.5, 2 * zeros) - t.mul_(math.pi) - kernel = (sinc(t) * winodd).view(1, 1, -1) - return kernel - - -def downsample2(x, zeros=56): - """ - Downsampling the input by 2 using sinc interpolation. - Smith, Julius, and Phil Gossett. "A flexible sampling-rate conversion method." - ICASSP'84. IEEE International Conference on Acoustics, Speech, and Signal Processing. - Vol. 9. IEEE, 1984. - """ - if x.shape[-1] % 2 != 0: - x = F.pad(x, (0, 1)) - xeven = x[..., ::2] - xodd = x[..., 1::2] - *other, time = xodd.shape - kernel = kernel_downsample2(zeros).to(x) - out = xeven + F.conv1d( - xodd.view(-1, 1, time), kernel, padding=zeros - )[..., :-1].view(*other, time) - return out.view(*other, -1).mul(0.5) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/preprocessing/denoiser/utils.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/preprocessing/denoiser/utils.py deleted file mode 100644 index 95b300c02b0f293e34bf05d376429e876efeccce..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/preprocessing/denoiser/utils.py +++ /dev/null @@ -1,176 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# All rights reserved. -# -# This source code is licensed under the license found in the -# LICENSE file in the root directory of this source tree. -# author: adefossez - -import functools -import inspect -import logging -import time -from contextlib import contextmanager - -logger = logging.getLogger(__name__) - -EPS = 1e-8 - - -def capture_init(init): - """capture_init. - - Decorate `__init__` with this, and you can then - recover the *args and **kwargs passed to it in `self._init_args_kwargs` - """ - @functools.wraps(init) - def __init__(self, *args, **kwargs): - self._init_args_kwargs = (args, kwargs) - init(self, *args, **kwargs) - - return __init__ - - -def deserialize_model(package, strict=False): - """deserialize_model. - - """ - klass = package['class'] - if strict: - model = klass(*package['args'], **package['kwargs']) - else: - sig = inspect.signature(klass) - kw = package['kwargs'] - for key in list(kw): - if key not in sig.parameters: - logger.warning("Dropping inexistant parameter %s", key) - del kw[key] - model = klass(*package['args'], **kw) - model.load_state_dict(package['state']) - return model - - -def copy_state(state): - return {k: v.cpu().clone() for k, v in state.items()} - - -def serialize_model(model): - args, kwargs = model._init_args_kwargs - state = copy_state(model.state_dict()) - return {"class": model.__class__, "args": args, "kwargs": kwargs, "state": state} - - -@contextmanager -def swap_state(model, state): - """ - Context manager that swaps the state of a model, e.g: - - # model is in old state - with swap_state(model, new_state): - # model in new state - # model back to old state - """ - old_state = copy_state(model.state_dict()) - model.load_state_dict(state) - try: - yield - finally: - model.load_state_dict(old_state) - - -def pull_metric(history, name): - out = [] - for metrics in history: - if name in metrics: - out.append(metrics[name]) - return out - - -class LogProgress: - """ - Sort of like tqdm but using log lines and not as real time. - Args: - - logger: logger obtained from `logging.getLogger`, - - iterable: iterable object to wrap - - updates (int): number of lines that will be printed, e.g. - if `updates=5`, log every 1/5th of the total length. - - total (int): length of the iterable, in case it does not support - `len`. - - name (str): prefix to use in the log. - - level: logging level (like `logging.INFO`). - """ - def __init__(self, - logger, - iterable, - updates=5, - total=None, - name="LogProgress", - level=logging.INFO): - self.iterable = iterable - self.total = total or len(iterable) - self.updates = updates - self.name = name - self.logger = logger - self.level = level - - def update(self, **infos): - self._infos = infos - - def __iter__(self): - self._iterator = iter(self.iterable) - self._index = -1 - self._infos = {} - self._begin = time.time() - return self - - def __next__(self): - self._index += 1 - try: - value = next(self._iterator) - except StopIteration: - raise - else: - return value - finally: - log_every = max(1, self.total // self.updates) - # logging is delayed by 1 it, in order to have the metrics from update - if self._index >= 1 and self._index % log_every == 0: - self._log() - - def _log(self): - self._speed = (1 + self._index) / (time.time() - self._begin) - infos = " | ".join(f"{k.capitalize()} {v}" for k, v in self._infos.items()) - if self._speed < 1e-4: - speed = "oo sec/it" - elif self._speed < 0.1: - speed = f"{1/self._speed:.1f} sec/it" - else: - speed = f"{self._speed:.1f} it/sec" - out = f"{self.name} | {self._index}/{self.total} | {speed}" - if infos: - out += " | " + infos - self.logger.log(self.level, out) - - -def colorize(text, color): - """ - Display text with some ANSI color in the terminal. - """ - code = f"\033[{color}m" - restore = "\033[0m" - return "".join([code, text, restore]) - - -def bold(text): - """ - Display text in bold in the terminal. - """ - return colorize(text, "1") - - -def cal_snr(lbl, est): - import torch - y = 10.0 * torch.log10( - torch.sum(lbl**2, dim=-1) / (torch.sum((est-lbl)**2, dim=-1) + EPS) + - EPS - ) - return y diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/preprocessing/get_common_voice_audio_manifest.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/preprocessing/get_common_voice_audio_manifest.py deleted file mode 100644 index 6026c57f829a6f2425029a067ec8e68cb4c23c07..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/preprocessing/get_common_voice_audio_manifest.py +++ /dev/null @@ -1,138 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import logging -from collections import defaultdict -from pathlib import Path -from typing import Dict, List, Tuple - -import numpy as np -import pandas as pd -import torchaudio -from examples.speech_to_text.data_utils import load_df_from_tsv, save_df_to_tsv -from tqdm import tqdm - -log = logging.getLogger(__name__) - -SPLITS = ["train", "dev", "test"] - - -def get_top_n( - root: Path, n_speakers: int = 10, min_n_tokens: int = 5 -) -> pd.DataFrame: - df = load_df_from_tsv(root / "validated.tsv") - df["n_tokens"] = [len(s.split()) for s in df["sentence"]] - df = df[df["n_tokens"] >= min_n_tokens] - df["n_frames"] = [ - torchaudio.info((root / "clips" / p).as_posix()).num_frames - for p in tqdm(df["path"]) - ] - df["id"] = [Path(p).stem for p in df["path"]] - total_duration_ms = df.groupby("client_id")["n_frames"].agg(["sum"]) - total_duration_ms = total_duration_ms.sort_values("sum", ascending=False) - - top_n_total_duration_ms = total_duration_ms.head(n_speakers) - top_n_client_ids = set(top_n_total_duration_ms.index.tolist()) - df_top_n = df[df["client_id"].isin(top_n_client_ids)] - return df_top_n - - -def get_splits( - df, train_split_ratio=0.99, speaker_in_all_splits=False, rand_seed=0 -) -> Tuple[Dict[str, str], List[str]]: - np.random.seed(rand_seed) - dev_split_ratio = (1. - train_split_ratio) / 3 - grouped = list(df.groupby("client_id")) - id_to_split = {} - for _, cur_df in tqdm(grouped): - cur_n_examples = len(cur_df) - if speaker_in_all_splits and cur_n_examples < 3: - continue - cur_n_train = int(cur_n_examples * train_split_ratio) - cur_n_dev = int(cur_n_examples * dev_split_ratio) - cur_n_test = cur_n_examples - cur_n_dev - cur_n_train - if speaker_in_all_splits and cur_n_dev * cur_n_test == 0: - cur_n_dev, cur_n_test = 1, 1 - cur_n_train = cur_n_examples - cur_n_dev - cur_n_test - cur_indices = cur_df.index.tolist() - cur_shuffled_indices = np.random.permutation(cur_n_examples) - cur_shuffled_indices = [cur_indices[i] for i in cur_shuffled_indices] - cur_indices_by_split = { - "train": cur_shuffled_indices[:cur_n_train], - "dev": cur_shuffled_indices[cur_n_train: cur_n_train + cur_n_dev], - "test": cur_shuffled_indices[cur_n_train + cur_n_dev:] - } - for split in SPLITS: - for i in cur_indices_by_split[split]: - id_ = df["id"].loc[i] - id_to_split[id_] = split - return id_to_split, sorted(df["client_id"].unique()) - - -def convert_to_wav(root: Path, filenames: List[str], target_sr=16_000): - out_root = root / "wav" - out_root.mkdir(exist_ok=True, parents=True) - print("Converting to WAV...") - for n in tqdm(filenames): - in_path = (root / "clips" / n).as_posix() - waveform, sr = torchaudio.load(in_path) - converted, converted_sr = torchaudio.sox_effects.apply_effects_tensor( - waveform, sr, [["rate", str(target_sr)], ["channels", "1"]] - ) - out_path = (out_root / Path(n).with_suffix(".wav").name).as_posix() - torchaudio.save(out_path, converted, converted_sr, encoding="PCM_S", - bits_per_sample=16) - - -def process(args): - data_root = Path(args.data_root).absolute() / args.lang - - # Generate TSV manifest - print("Generating manifest...") - - df_top_n = get_top_n(data_root) - id_to_split, speakers = get_splits(df_top_n) - - if args.convert_to_wav: - convert_to_wav(data_root, df_top_n["path"].tolist()) - - manifest_by_split = {split: defaultdict(list) for split in SPLITS} - for sample in tqdm(df_top_n.to_dict(orient="index").values()): - sample_id = sample["id"] - split = id_to_split[sample_id] - manifest_by_split[split]["id"].append(sample_id) - if args.convert_to_wav: - audio_path = data_root / "wav" / f"{sample_id}.wav" - else: - audio_path = data_root / "clips" / f"{sample_id}.mp3" - manifest_by_split[split]["audio"].append(audio_path.as_posix()) - manifest_by_split[split]["n_frames"].append(sample["n_frames"]) - manifest_by_split[split]["tgt_text"].append(sample["sentence"]) - manifest_by_split[split]["speaker"].append(sample["client_id"]) - manifest_by_split[split]["src_text"].append(sample["sentence"]) - - output_root = Path(args.output_manifest_root).absolute() - output_root.mkdir(parents=True, exist_ok=True) - for split in SPLITS: - save_df_to_tsv( - pd.DataFrame.from_dict(manifest_by_split[split]), - output_root / f"{split}.audio.tsv" - ) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--data-root", "-d", required=True, type=str) - parser.add_argument("--output-manifest-root", "-m", required=True, type=str) - parser.add_argument("--lang", "-l", required=True, type=str) - parser.add_argument("--convert-to-wav", action="store_true") - args = parser.parse_args() - - process(args) - - -if __name__ == "__main__": - main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/preprocessing/get_feature_manifest.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/preprocessing/get_feature_manifest.py deleted file mode 100644 index f0a1ff88deebd4d645e582ac0e95eb224519b895..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/preprocessing/get_feature_manifest.py +++ /dev/null @@ -1,265 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import logging -import shutil -from collections import Counter, defaultdict -from pathlib import Path -from tempfile import NamedTemporaryFile - -import pandas as pd -import torchaudio -from examples.speech_synthesis.data_utils import ( - extract_energy, - extract_logmel_spectrogram, - extract_pitch, - get_feature_value_min_max, - get_global_cmvn, - get_mfa_alignment, - get_unit_alignment, - ipa_phonemize, -) -from examples.speech_to_text.data_utils import ( - create_zip, - gen_config_yaml, - gen_vocab, - get_zip_manifest, - load_tsv_to_dicts, - save_df_to_tsv, -) -from fairseq.data.audio.audio_utils import convert_waveform -from tqdm import tqdm - -log = logging.getLogger(__name__) - - -def process(args): - assert "train" in args.splits - out_root = Path(args.output_root).absolute() - out_root.mkdir(exist_ok=True) - - print("Fetching data...") - audio_manifest_root = Path(args.audio_manifest_root).absolute() - samples = [] - for s in args.splits: - for e in load_tsv_to_dicts(audio_manifest_root / f"{s}.audio.tsv"): - e["split"] = s - samples.append(e) - sample_ids = [s["id"] for s in samples] - - # Get alignment info - id_to_alignment = None - if args.textgrid_zip is not None: - assert args.id_to_units_tsv is None - id_to_alignment = get_mfa_alignment( - args.textgrid_zip, sample_ids, args.sample_rate, args.hop_length - ) - elif args.id_to_units_tsv is not None: - # assume identical hop length on the unit sequence - id_to_alignment = get_unit_alignment(args.id_to_units_tsv, sample_ids) - - # Extract features and pack features into ZIP - feature_name = "logmelspec80" - zip_path = out_root / f"{feature_name}.zip" - pitch_zip_path = out_root / "pitch.zip" - energy_zip_path = out_root / "energy.zip" - gcmvn_npz_path = out_root / "gcmvn_stats.npz" - if zip_path.exists() and gcmvn_npz_path.exists(): - print(f"{zip_path} and {gcmvn_npz_path} exist.") - else: - feature_root = out_root / feature_name - feature_root.mkdir(exist_ok=True) - pitch_root = out_root / "pitch" - energy_root = out_root / "energy" - if args.add_fastspeech_targets: - pitch_root.mkdir(exist_ok=True) - energy_root.mkdir(exist_ok=True) - print("Extracting Mel spectrogram features...") - for sample in tqdm(samples): - waveform, sample_rate = torchaudio.load(sample["audio"]) - waveform, sample_rate = convert_waveform( - waveform, sample_rate, normalize_volume=args.normalize_volume, - to_sample_rate=args.sample_rate - ) - sample_id = sample["id"] - target_length = None - if id_to_alignment is not None: - a = id_to_alignment[sample_id] - target_length = sum(a.frame_durations) - if a.start_sec is not None and a.end_sec is not None: - start_frame = int(a.start_sec * sample_rate) - end_frame = int(a.end_sec * sample_rate) - waveform = waveform[:, start_frame: end_frame] - extract_logmel_spectrogram( - waveform, sample_rate, feature_root / f"{sample_id}.npy", - win_length=args.win_length, hop_length=args.hop_length, - n_fft=args.n_fft, n_mels=args.n_mels, f_min=args.f_min, - f_max=args.f_max, target_length=target_length - ) - if args.add_fastspeech_targets: - assert id_to_alignment is not None - extract_pitch( - waveform, sample_rate, pitch_root / f"{sample_id}.npy", - hop_length=args.hop_length, log_scale=True, - phoneme_durations=id_to_alignment[sample_id].frame_durations - ) - extract_energy( - waveform, energy_root / f"{sample_id}.npy", - hop_length=args.hop_length, n_fft=args.n_fft, - log_scale=True, - phoneme_durations=id_to_alignment[sample_id].frame_durations - ) - print("ZIPing features...") - create_zip(feature_root, zip_path) - get_global_cmvn(feature_root, gcmvn_npz_path) - shutil.rmtree(feature_root) - if args.add_fastspeech_targets: - create_zip(pitch_root, pitch_zip_path) - shutil.rmtree(pitch_root) - create_zip(energy_root, energy_zip_path) - shutil.rmtree(energy_root) - - print("Fetching ZIP manifest...") - audio_paths, audio_lengths = get_zip_manifest(zip_path) - pitch_paths, pitch_lengths, energy_paths, energy_lengths = [None] * 4 - if args.add_fastspeech_targets: - pitch_paths, pitch_lengths = get_zip_manifest(pitch_zip_path) - energy_paths, energy_lengths = get_zip_manifest(energy_zip_path) - # Generate TSV manifest - print("Generating manifest...") - id_to_cer = None - if args.cer_threshold is not None: - assert Path(args.cer_tsv_path).is_file() - id_to_cer = { - x["id"]: x["uer"] for x in load_tsv_to_dicts(args.cer_tsv_path) - } - manifest_by_split = {split: defaultdict(list) for split in args.splits} - for sample in tqdm(samples): - sample_id, split = sample["id"], sample["split"] - - if args.snr_threshold is not None and "snr" in sample \ - and sample["snr"] < args.snr_threshold: - continue - if args.cer_threshold is not None \ - and id_to_cer[sample_id] > args.cer_threhold: - continue - - normalized_utt = sample["tgt_text"] - if id_to_alignment is not None: - normalized_utt = " ".join(id_to_alignment[sample_id].tokens) - elif args.ipa_vocab: - normalized_utt = ipa_phonemize( - normalized_utt, lang=args.lang, use_g2p=args.use_g2p - ) - manifest_by_split[split]["id"].append(sample_id) - manifest_by_split[split]["audio"].append(audio_paths[sample_id]) - manifest_by_split[split]["n_frames"].append(audio_lengths[sample_id]) - manifest_by_split[split]["tgt_text"].append(normalized_utt) - manifest_by_split[split]["speaker"].append(sample["speaker"]) - manifest_by_split[split]["src_text"].append(sample["src_text"]) - if args.add_fastspeech_targets: - assert id_to_alignment is not None - duration = " ".join( - str(d) for d in id_to_alignment[sample_id].frame_durations - ) - manifest_by_split[split]["duration"].append(duration) - manifest_by_split[split]["pitch"].append(pitch_paths[sample_id]) - manifest_by_split[split]["energy"].append(energy_paths[sample_id]) - for split in args.splits: - save_df_to_tsv( - pd.DataFrame.from_dict(manifest_by_split[split]), - out_root / f"{split}.tsv" - ) - # Generate vocab - vocab_name, spm_filename = None, None - if id_to_alignment is not None or args.ipa_vocab: - vocab = Counter() - for t in manifest_by_split["train"]["tgt_text"]: - vocab.update(t.split(" ")) - vocab_name = "vocab.txt" - with open(out_root / vocab_name, "w") as f: - for s, c in vocab.most_common(): - f.write(f"{s} {c}\n") - else: - spm_filename_prefix = "spm_char" - spm_filename = f"{spm_filename_prefix}.model" - with NamedTemporaryFile(mode="w") as f: - for t in manifest_by_split["train"]["tgt_text"]: - f.write(t + "\n") - f.flush() # needed to ensure gen_vocab sees dumped text - gen_vocab(Path(f.name), out_root / spm_filename_prefix, "char") - # Generate speaker list - speakers = sorted({sample["speaker"] for sample in samples}) - speakers_path = out_root / "speakers.txt" - with open(speakers_path, "w") as f: - for speaker in speakers: - f.write(f"{speaker}\n") - # Generate config YAML - win_len_t = args.win_length / args.sample_rate - hop_len_t = args.hop_length / args.sample_rate - extra = { - "sample_rate": args.sample_rate, - "features": { - "type": "spectrogram+melscale+log", - "eps": 1e-5, "n_mels": args.n_mels, "n_fft": args.n_fft, - "window_fn": "hann", "win_length": args.win_length, - "hop_length": args.hop_length, "sample_rate": args.sample_rate, - "win_len_t": win_len_t, "hop_len_t": hop_len_t, - "f_min": args.f_min, "f_max": args.f_max, - "n_stft": args.n_fft // 2 + 1 - } - } - if len(speakers) > 1: - extra["speaker_set_filename"] = "speakers.txt" - if args.add_fastspeech_targets: - pitch_min, pitch_max = get_feature_value_min_max( - [(out_root / n).as_posix() for n in pitch_paths.values()] - ) - energy_min, energy_max = get_feature_value_min_max( - [(out_root / n).as_posix() for n in energy_paths.values()] - ) - extra["features"]["pitch_min"] = pitch_min - extra["features"]["pitch_max"] = pitch_max - extra["features"]["energy_min"] = energy_min - extra["features"]["energy_max"] = energy_max - gen_config_yaml( - out_root, spm_filename=spm_filename, vocab_name=vocab_name, - audio_root=out_root.as_posix(), input_channels=None, - input_feat_per_channel=None, specaugment_policy=None, - cmvn_type="global", gcmvn_path=gcmvn_npz_path, extra=extra - ) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--audio-manifest-root", "-m", required=True, type=str) - parser.add_argument("--output-root", "-o", required=True, type=str) - parser.add_argument("--splits", "-s", type=str, nargs="+", - default=["train", "dev", "test"]) - parser.add_argument("--ipa-vocab", action="store_true") - parser.add_argument("--use-g2p", action="store_true") - parser.add_argument("--lang", type=str, default="en-us") - parser.add_argument("--win-length", type=int, default=1024) - parser.add_argument("--hop-length", type=int, default=256) - parser.add_argument("--n-fft", type=int, default=1024) - parser.add_argument("--n-mels", type=int, default=80) - parser.add_argument("--f-min", type=int, default=20) - parser.add_argument("--f-max", type=int, default=8000) - parser.add_argument("--sample-rate", type=int, default=22050) - parser.add_argument("--normalize-volume", "-n", action="store_true") - parser.add_argument("--textgrid-zip", type=str, default=None) - parser.add_argument("--id-to-units-tsv", type=str, default=None) - parser.add_argument("--add-fastspeech-targets", action="store_true") - parser.add_argument("--snr-threshold", type=float, default=None) - parser.add_argument("--cer-threshold", type=float, default=None) - parser.add_argument("--cer-tsv-path", type=str, default="") - args = parser.parse_args() - - process(args) - - -if __name__ == "__main__": - main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/preprocessing/get_ljspeech_audio_manifest.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/preprocessing/get_ljspeech_audio_manifest.py deleted file mode 100644 index 0ffc643b9ca6a171a678bbe264a3bc4f7663348a..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/preprocessing/get_ljspeech_audio_manifest.py +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import logging -from collections import defaultdict -from pathlib import Path - -import pandas as pd -from examples.speech_to_text.data_utils import save_df_to_tsv -from torchaudio.datasets import LJSPEECH -from tqdm import tqdm - -log = logging.getLogger(__name__) - -SPLITS = ["train", "dev", "test"] - - -def process(args): - out_root = Path(args.output_data_root).absolute() - out_root.mkdir(parents=True, exist_ok=True) - - # Generate TSV manifest - print("Generating manifest...") - # following FastSpeech's splits - dataset = LJSPEECH(out_root.as_posix(), download=True) - id_to_split = {} - for x in dataset._flist: - id_ = x[0] - speaker = id_.split("-")[0] - id_to_split[id_] = { - "LJ001": "test", "LJ002": "test", "LJ003": "dev" - }.get(speaker, "train") - manifest_by_split = {split: defaultdict(list) for split in SPLITS} - progress = tqdm(enumerate(dataset), total=len(dataset)) - for i, (waveform, _, utt, normalized_utt) in progress: - sample_id = dataset._flist[i][0] - split = id_to_split[sample_id] - manifest_by_split[split]["id"].append(sample_id) - audio_path = f"{dataset._path}/{sample_id}.wav" - manifest_by_split[split]["audio"].append(audio_path) - manifest_by_split[split]["n_frames"].append(len(waveform[0])) - manifest_by_split[split]["tgt_text"].append(normalized_utt) - manifest_by_split[split]["speaker"].append("ljspeech") - manifest_by_split[split]["src_text"].append(utt) - - manifest_root = Path(args.output_manifest_root).absolute() - manifest_root.mkdir(parents=True, exist_ok=True) - for split in SPLITS: - save_df_to_tsv( - pd.DataFrame.from_dict(manifest_by_split[split]), - manifest_root / f"{split}.audio.tsv" - ) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--output-data-root", "-d", required=True, type=str) - parser.add_argument("--output-manifest-root", "-m", required=True, type=str) - args = parser.parse_args() - - process(args) - - -if __name__ == "__main__": - main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/preprocessing/get_speaker_embedding.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/preprocessing/get_speaker_embedding.py deleted file mode 100644 index a70312049cd5060ad694617f1a100c15b5ff2b80..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/preprocessing/get_speaker_embedding.py +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - - -import argparse -from collections import defaultdict -from itertools import chain -from pathlib import Path - -import numpy as np -import torchaudio -import torchaudio.sox_effects as ta_sox -import yaml -from examples.speech_synthesis.preprocessing.speaker_embedder import SpkrEmbedder -from examples.speech_to_text.data_utils import load_tsv_to_dicts -from tqdm import tqdm - - -def extract_embedding(audio_path, embedder): - wav, sr = torchaudio.load(audio_path) # 2D - if sr != embedder.RATE: - wav, sr = ta_sox.apply_effects_tensor( - wav, sr, [["rate", str(embedder.RATE)]] - ) - try: - emb = embedder([wav[0].cuda().float()]).cpu().numpy() - except RuntimeError: - emb = None - return emb - - -def process(args): - print("Fetching data...") - raw_manifest_root = Path(args.raw_manifest_root).absolute() - samples = [load_tsv_to_dicts(raw_manifest_root / (s + ".tsv")) - for s in args.splits] - samples = list(chain(*samples)) - with open(args.config, "r") as f: - config = yaml.load(f, Loader=yaml.FullLoader) - with open(f"{config['audio_root']}/{config['speaker_set_filename']}") as f: - speaker_to_id = {r.strip(): i for i, r in enumerate(f)} - - embedder = SpkrEmbedder(args.ckpt).cuda() - speaker_to_cnt = defaultdict(float) - speaker_to_emb = defaultdict(float) - for sample in tqdm(samples, desc="extract emb"): - emb = extract_embedding(sample["audio"], embedder) - if emb is not None: - speaker_to_cnt[sample["speaker"]] += 1 - speaker_to_emb[sample["speaker"]] += emb - if len(speaker_to_emb) != len(speaker_to_id): - missed = set(speaker_to_id) - set(speaker_to_emb.keys()) - print( - f"WARNING: missing embeddings for {len(missed)} speaker:\n{missed}" - ) - speaker_emb_mat = np.zeros((len(speaker_to_id), len(emb)), float) - for speaker in speaker_to_emb: - idx = speaker_to_id[speaker] - emb = speaker_to_emb[speaker] - cnt = speaker_to_cnt[speaker] - speaker_emb_mat[idx, :] = emb / cnt - speaker_emb_name = "speaker_emb.npy" - speaker_emb_path = f"{config['audio_root']}/{speaker_emb_name}" - np.save(speaker_emb_path, speaker_emb_mat) - config["speaker_emb_filename"] = speaker_emb_name - - with open(args.new_config, "w") as f: - yaml.dump(config, f) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--raw-manifest-root", "-m", required=True, type=str) - parser.add_argument("--splits", "-s", type=str, nargs="+", - default=["train"]) - parser.add_argument("--config", "-c", required=True, type=str) - parser.add_argument("--new-config", "-n", required=True, type=str) - parser.add_argument("--ckpt", required=True, type=str, - help="speaker embedder checkpoint") - args = parser.parse_args() - - process(args) - - -if __name__ == "__main__": - main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/preprocessing/get_vctk_audio_manifest.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/preprocessing/get_vctk_audio_manifest.py deleted file mode 100644 index 89481766b732e7566912acb0833e2fcfb47e45f6..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/preprocessing/get_vctk_audio_manifest.py +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import logging -import re -from collections import defaultdict -from pathlib import Path - -import numpy as np -import pandas as pd -from examples.speech_to_text.data_utils import save_df_to_tsv -from torchaudio.datasets import VCTK -from tqdm import tqdm - -log = logging.getLogger(__name__) - -SPLITS = ["train", "dev", "test"] - - -def normalize_text(text): - return re.sub(r"[^a-zA-Z.?!,'\- ]", '', text) - - -def process(args): - out_root = Path(args.output_data_root).absolute() - out_root.mkdir(parents=True, exist_ok=True) - - # Generate TSV manifest - print("Generating manifest...") - dataset = VCTK(out_root.as_posix(), download=False) - ids = list(dataset._walker) - np.random.seed(args.seed) - np.random.shuffle(ids) - n_train = len(ids) - args.n_dev - args.n_test - _split = ["train"] * n_train + ["dev"] * args.n_dev + ["test"] * args.n_test - id_to_split = dict(zip(ids, _split)) - manifest_by_split = {split: defaultdict(list) for split in SPLITS} - progress = tqdm(enumerate(dataset), total=len(dataset)) - for i, (waveform, _, text, speaker_id, _) in progress: - sample_id = dataset._walker[i] - _split = id_to_split[sample_id] - audio_dir = Path(dataset._path) / dataset._folder_audio / speaker_id - audio_path = audio_dir / f"{sample_id}.wav" - text = normalize_text(text) - manifest_by_split[_split]["id"].append(sample_id) - manifest_by_split[_split]["audio"].append(audio_path.as_posix()) - manifest_by_split[_split]["n_frames"].append(len(waveform[0])) - manifest_by_split[_split]["tgt_text"].append(text) - manifest_by_split[_split]["speaker"].append(speaker_id) - manifest_by_split[_split]["src_text"].append(text) - - manifest_root = Path(args.output_manifest_root).absolute() - manifest_root.mkdir(parents=True, exist_ok=True) - for _split in SPLITS: - save_df_to_tsv( - pd.DataFrame.from_dict(manifest_by_split[_split]), - manifest_root / f"{_split}.audio.tsv" - ) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--output-data-root", "-d", required=True, type=str) - parser.add_argument("--output-manifest-root", "-m", required=True, type=str) - parser.add_argument("--n-dev", default=50, type=int) - parser.add_argument("--n-test", default=100, type=int) - parser.add_argument("--seed", "-s", default=1234, type=int) - args = parser.parse_args() - - process(args) - - -if __name__ == "__main__": - main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/preprocessing/speaker_embedder/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/preprocessing/speaker_embedder/__init__.py deleted file mode 100644 index 8ecf03b73cd478e47ceea25595b780038cb5a12f..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/preprocessing/speaker_embedder/__init__.py +++ /dev/null @@ -1,134 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - - -import librosa -import torch -import torch.nn as nn -import torch.nn.functional as F -import torch.utils.data -import torchaudio - -EMBEDDER_PARAMS = { - 'num_mels': 40, - 'n_fft': 512, - 'emb_dim': 256, - 'lstm_hidden': 768, - 'lstm_layers': 3, - 'window': 80, - 'stride': 40, -} - - -def set_requires_grad(nets, requires_grad=False): - """Set requies_grad=Fasle for all the networks to avoid unnecessary - computations - Parameters: - nets (network list) -- a list of networks - requires_grad (bool) -- whether the networks require gradients or not - """ - if not isinstance(nets, list): - nets = [nets] - for net in nets: - if net is not None: - for param in net.parameters(): - param.requires_grad = requires_grad - - -class LinearNorm(nn.Module): - def __init__(self, hp): - super(LinearNorm, self).__init__() - self.linear_layer = nn.Linear(hp["lstm_hidden"], hp["emb_dim"]) - - def forward(self, x): - return self.linear_layer(x) - - -class SpeechEmbedder(nn.Module): - def __init__(self, hp): - super(SpeechEmbedder, self).__init__() - self.lstm = nn.LSTM(hp["num_mels"], - hp["lstm_hidden"], - num_layers=hp["lstm_layers"], - batch_first=True) - self.proj = LinearNorm(hp) - self.hp = hp - - def forward(self, mel): - # (num_mels, T) -> (num_mels, T', window) - mels = mel.unfold(1, self.hp["window"], self.hp["stride"]) - mels = mels.permute(1, 2, 0) # (T', window, num_mels) - x, _ = self.lstm(mels) # (T', window, lstm_hidden) - x = x[:, -1, :] # (T', lstm_hidden), use last frame only - x = self.proj(x) # (T', emb_dim) - x = x / torch.norm(x, p=2, dim=1, keepdim=True) # (T', emb_dim) - - x = x.mean(dim=0) - if x.norm(p=2) != 0: - x = x / x.norm(p=2) - return x - - -class SpkrEmbedder(nn.Module): - RATE = 16000 - - def __init__( - self, - embedder_path, - embedder_params=EMBEDDER_PARAMS, - rate=16000, - hop_length=160, - win_length=400, - pad=False, - ): - super(SpkrEmbedder, self).__init__() - embedder_pt = torch.load(embedder_path, map_location="cpu") - self.embedder = SpeechEmbedder(embedder_params) - self.embedder.load_state_dict(embedder_pt) - self.embedder.eval() - set_requires_grad(self.embedder, requires_grad=False) - self.embedder_params = embedder_params - - self.register_buffer('mel_basis', torch.from_numpy( - librosa.filters.mel( - sr=self.RATE, - n_fft=self.embedder_params["n_fft"], - n_mels=self.embedder_params["num_mels"]) - ) - ) - - self.resample = None - if rate != self.RATE: - self.resample = torchaudio.transforms.Resample(rate, self.RATE) - self.hop_length = hop_length - self.win_length = win_length - self.pad = pad - - def get_mel(self, y): - if self.pad and y.shape[-1] < 14000: - y = F.pad(y, (0, 14000 - y.shape[-1])) - - window = torch.hann_window(self.win_length).to(y) - y = torch.stft(y, n_fft=self.embedder_params["n_fft"], - hop_length=self.hop_length, - win_length=self.win_length, - window=window) - magnitudes = torch.norm(y, dim=-1, p=2) ** 2 - mel = torch.log10(self.mel_basis @ magnitudes + 1e-6) - return mel - - def forward(self, inputs): - dvecs = [] - for wav in inputs: - mel = self.get_mel(wav) - if mel.dim() == 3: - mel = mel.squeeze(0) - dvecs += [self.embedder(mel)] - dvecs = torch.stack(dvecs) - - dvec = torch.mean(dvecs, dim=0) - dvec = dvec / torch.norm(dvec) - - return dvec diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/preprocessing/vad/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/preprocessing/vad/__init__.py deleted file mode 100644 index d43f816bddcf5cb3ad29c00b3203eeffb7ddde7c..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/preprocessing/vad/__init__.py +++ /dev/null @@ -1,193 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - - -import collections -import contextlib -import wave - -try: - import webrtcvad -except ImportError: - raise ImportError("Please install py-webrtcvad: pip install webrtcvad") -import argparse -import logging -import os - -from tqdm import tqdm - -AUDIO_SUFFIX = '.wav' -FS_MS = 30 -SCALE = 6e-5 -THRESHOLD = 0.3 - - -def read_wave(path): - """Reads a .wav file. - Takes the path, and returns (PCM audio data, sample rate). - """ - with contextlib.closing(wave.open(path, 'rb')) as wf: - num_channels = wf.getnchannels() - assert num_channels == 1 - sample_width = wf.getsampwidth() - assert sample_width == 2 - sample_rate = wf.getframerate() - assert sample_rate in (8000, 16000, 32000, 48000) - pcm_data = wf.readframes(wf.getnframes()) - return pcm_data, sample_rate - - -def write_wave(path, audio, sample_rate): - """Writes a .wav file. - Takes path, PCM audio data, and sample rate. - """ - with contextlib.closing(wave.open(path, 'wb')) as wf: - wf.setnchannels(1) - wf.setsampwidth(2) - wf.setframerate(sample_rate) - wf.writeframes(audio) - - -class Frame(object): - """Represents a "frame" of audio data.""" - def __init__(self, bytes, timestamp, duration): - self.bytes = bytes - self.timestamp = timestamp - self.duration = duration - - -def frame_generator(frame_duration_ms, audio, sample_rate): - """Generates audio frames from PCM audio data. - Takes the desired frame duration in milliseconds, the PCM data, and - the sample rate. - Yields Frames of the requested duration. - """ - n = int(sample_rate * (frame_duration_ms / 1000.0) * 2) - offset = 0 - timestamp = 0.0 - duration = (float(n) / sample_rate) / 2.0 - while offset + n < len(audio): - yield Frame(audio[offset:offset + n], timestamp, duration) - timestamp += duration - offset += n - - -def vad_collector(sample_rate, frame_duration_ms, - padding_duration_ms, vad, frames): - """Filters out non-voiced audio frames. - Given a webrtcvad.Vad and a source of audio frames, yields only - the voiced audio. - Uses a padded, sliding window algorithm over the audio frames. - When more than 90% of the frames in the window are voiced (as - reported by the VAD), the collector triggers and begins yielding - audio frames. Then the collector waits until 90% of the frames in - the window are unvoiced to detrigger. - The window is padded at the front and back to provide a small - amount of silence or the beginnings/endings of speech around the - voiced frames. - Arguments: - sample_rate - The audio sample rate, in Hz. - frame_duration_ms - The frame duration in milliseconds. - padding_duration_ms - The amount to pad the window, in milliseconds. - vad - An instance of webrtcvad.Vad. - frames - a source of audio frames (sequence or generator). - Returns: A generator that yields PCM audio data. - """ - num_padding_frames = int(padding_duration_ms / frame_duration_ms) - # We use a deque for our sliding window/ring buffer. - ring_buffer = collections.deque(maxlen=num_padding_frames) - # We have two states: TRIGGERED and NOTTRIGGERED. We start in the - # NOTTRIGGERED state. - triggered = False - - voiced_frames = [] - for frame in frames: - is_speech = vad.is_speech(frame.bytes, sample_rate) - - # sys.stdout.write('1' if is_speech else '0') - if not triggered: - ring_buffer.append((frame, is_speech)) - num_voiced = len([f for f, speech in ring_buffer if speech]) - # If we're NOTTRIGGERED and more than 90% of the frames in - # the ring buffer are voiced frames, then enter the - # TRIGGERED state. - if num_voiced > 0.9 * ring_buffer.maxlen: - triggered = True - # We want to yield all the audio we see from now until - # we are NOTTRIGGERED, but we have to start with the - # audio that's already in the ring buffer. - for f, _ in ring_buffer: - voiced_frames.append(f) - ring_buffer.clear() - else: - # We're in the TRIGGERED state, so collect the audio data - # and add it to the ring buffer. - voiced_frames.append(frame) - ring_buffer.append((frame, is_speech)) - num_unvoiced = len([f for f, speech in ring_buffer if not speech]) - # If more than 90% of the frames in the ring buffer are - # unvoiced, then enter NOTTRIGGERED and yield whatever - # audio we've collected. - if num_unvoiced > 0.9 * ring_buffer.maxlen: - triggered = False - yield [b''.join([f.bytes for f in voiced_frames]), - voiced_frames[0].timestamp, voiced_frames[-1].timestamp] - ring_buffer.clear() - voiced_frames = [] - # If we have any leftover voiced audio when we run out of input, - # yield it. - if voiced_frames: - yield [b''.join([f.bytes for f in voiced_frames]), - voiced_frames[0].timestamp, voiced_frames[-1].timestamp] - - -def main(args): - # create output folder - try: - cmd = f"mkdir -p {args.out_path}" - os.system(cmd) - except Exception: - logging.error("Can not create output folder") - exit(-1) - - # build vad object - vad = webrtcvad.Vad(int(args.agg)) - # iterating over wavs in dir - for file in tqdm(os.listdir(args.in_path)): - if file.endswith(AUDIO_SUFFIX): - audio_inpath = os.path.join(args.in_path, file) - audio_outpath = os.path.join(args.out_path, file) - audio, sample_rate = read_wave(audio_inpath) - frames = frame_generator(FS_MS, audio, sample_rate) - frames = list(frames) - segments = vad_collector(sample_rate, FS_MS, 300, vad, frames) - merge_segments = list() - timestamp_start = 0.0 - timestamp_end = 0.0 - # removing start, end, and long sequences of sils - for i, segment in enumerate(segments): - merge_segments.append(segment[0]) - if i and timestamp_start: - sil_duration = segment[1] - timestamp_end - if sil_duration > THRESHOLD: - merge_segments.append(int(THRESHOLD / SCALE)*(b'\x00')) - else: - merge_segments.append(int((sil_duration / SCALE))*(b'\x00')) - timestamp_start = segment[1] - timestamp_end = segment[2] - segment = b''.join(merge_segments) - write_wave(audio_outpath, segment, sample_rate) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Apply vad to a file of fils.') - parser.add_argument('in_path', type=str, help='Path to the input files') - parser.add_argument('out_path', type=str, - help='Path to save the processed files') - parser.add_argument('--agg', type=int, default=3, - help='The level of aggressiveness of the VAD: [0-3]') - args = parser.parse_args() - - main(args) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/utils.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/utils.py deleted file mode 100644 index b2a9a4fbfe206c9da59d4b016f283f3602463fa0..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/utils.py +++ /dev/null @@ -1,98 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import numpy as np -import torch -import torchaudio -from fairseq.tasks.text_to_speech import batch_compute_distortion, compute_rms_dist -from scipy.interpolate import interp1d - - -def batch_mel_spectral_distortion( - y1, y2, sr, normalize_type="path", mel_fn=None -): - """ - https://arxiv.org/pdf/2011.03568.pdf - - Same as Mel Cepstral Distortion, but computed on log-mel spectrograms. - """ - if mel_fn is None or mel_fn.sample_rate != sr: - mel_fn = torchaudio.transforms.MelSpectrogram( - sr, n_fft=int(0.05 * sr), win_length=int(0.05 * sr), - hop_length=int(0.0125 * sr), f_min=20, n_mels=80, - window_fn=torch.hann_window - ).to(y1[0].device) - offset = 1e-6 - return batch_compute_distortion( - y1, y2, sr, lambda y: torch.log(mel_fn(y) + offset).transpose(-1, -2), - compute_rms_dist, normalize_type - ) - - -# This code is based on -# "https://github.com/bastibe/MAPS-Scripts/blob/master/helper.py" -def _same_t_in_true_and_est(func): - def new_func(true_t, true_f, est_t, est_f): - assert type(true_t) is np.ndarray - assert type(true_f) is np.ndarray - assert type(est_t) is np.ndarray - assert type(est_f) is np.ndarray - - interpolated_f = interp1d( - est_t, est_f, bounds_error=False, kind='nearest', fill_value=0 - )(true_t) - return func(true_t, true_f, true_t, interpolated_f) - - return new_func - - -@_same_t_in_true_and_est -def gross_pitch_error(true_t, true_f, est_t, est_f): - """The relative frequency in percent of pitch estimates that are - outside a threshold around the true pitch. Only frames that are - considered pitched by both the ground truth and the estimator (if - applicable) are considered. - """ - - correct_frames = _true_voiced_frames(true_t, true_f, est_t, est_f) - gross_pitch_error_frames = _gross_pitch_error_frames( - true_t, true_f, est_t, est_f - ) - return np.sum(gross_pitch_error_frames) / np.sum(correct_frames) - - -def _gross_pitch_error_frames(true_t, true_f, est_t, est_f, eps=1e-8): - voiced_frames = _true_voiced_frames(true_t, true_f, est_t, est_f) - true_f_p_eps = [x + eps for x in true_f] - pitch_error_frames = np.abs(est_f / true_f_p_eps - 1) > 0.2 - return voiced_frames & pitch_error_frames - - -def _true_voiced_frames(true_t, true_f, est_t, est_f): - return (est_f != 0) & (true_f != 0) - - -def _voicing_decision_error_frames(true_t, true_f, est_t, est_f): - return (est_f != 0) != (true_f != 0) - - -@_same_t_in_true_and_est -def f0_frame_error(true_t, true_f, est_t, est_f): - gross_pitch_error_frames = _gross_pitch_error_frames( - true_t, true_f, est_t, est_f - ) - voicing_decision_error_frames = _voicing_decision_error_frames( - true_t, true_f, est_t, est_f - ) - return (np.sum(gross_pitch_error_frames) + - np.sum(voicing_decision_error_frames)) / (len(true_t)) - - -@_same_t_in_true_and_est -def voicing_decision_error(true_t, true_f, est_t, est_f): - voicing_decision_error_frames = _voicing_decision_error_frames( - true_t, true_f, est_t, est_f - ) - return np.sum(voicing_decision_error_frames) / (len(true_t)) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/README.md b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/README.md deleted file mode 100644 index c1aa11929a434a0d146dcfa05fc067d0fac1b310..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/README.md +++ /dev/null @@ -1,51 +0,0 @@ -# Joint Speech Text training in Fairseq -An extension of Fairseq s2t project with the speech to text task enhanced by the co-trained text to text mapping task. More details about Fairseq s2t can be found [here](../speech_to_text/README.md) - -## Examples -Examples of speech text joint training in fairseq -- [English-to-German MuST-C model](docs/ende-mustc.md) -- [IWSLT 2021 Multilingual Speech Translation](docs/iwslt2021.md) -- [Speech Text Joint Pre-training ](docs/pre-training.md) -## Citation -Please cite as: -``` -@inproceedings{Tang2022UnifiedSP, - title={Unified Speech-Text Pre-training for Speech Translation and Recognition}, - author={Yun Tang and Hongyu Gong and Ning Dong and Changhan Wang and Wei-Ning Hsu and Jiatao Gu and Alexei Baevski and Xian Li and Abdelrahman Mohamed and Michael Auli and Juan Miguel Pino}, - booktitle={ACL}, - year={2022} -} -@inproceedings{Tang2021IST, - title = {Improving Speech Translation by Understanding and Learning from the Auxiliary Text Translation Task}, - author = {Yun Tang and Juan Pino and Xian Li and Changhan Wang and Dmitriy Genzel}, - booktitle = {ACL}, - year = {2021}, -} - -@inproceedings{Tang2021FST, - title = {FST: the FAIR Speech Translation System for the IWSLT21 Multilingual Shared Task}, - author = {Yun Tang and Hongyu Gong and Xian Li and Changhan Wang and Juan Pino and Holger Schwenk and Naman Goyal}, - booktitle = {IWSLT}, - year = {2021}, -} -@inproceedings{Tang2021AGM, - title={A General Multi-Task Learning Framework to Leverage Text Data for Speech to Text Tasks}, - author={Yun Tang and J. Pino and Changhan Wang and Xutai Ma and Dmitriy Genzel}, - booktitle={ICASSP}, - year={2021} -} - -@inproceedings{wang2020fairseqs2t, - title = {fairseq S2T: Fast Speech-to-Text Modeling with fairseq}, - author = {Changhan Wang and Yun Tang and Xutai Ma and Anne Wu and Dmytro Okhonko and Juan Pino}, - booktitle = {Proceedings of the 2020 Conference of the Asian Chapter of the Association for Computational Linguistics (AACL): System Demonstrations}, - year = {2020}, -} - -@inproceedings{ott2019fairseq, - title = {fairseq: A Fast, Extensible Toolkit for Sequence Modeling}, - author = {Myle Ott and Sergey Edunov and Alexei Baevski and Angela Fan and Sam Gross and Nathan Ng and David Grangier and Michael Auli}, - booktitle = {Proceedings of NAACL-HLT 2019: Demonstrations}, - year = {2019}, -} -``` diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/__init__.py deleted file mode 100644 index 0c959f6f9a198f6ed7940553756e5cc35512a8d5..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from . import criterions, models, tasks # noqa diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/configs/mustc_noise.list b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/configs/mustc_noise.list deleted file mode 100644 index 02eeac4e009f77b765004272f59a1618214da18d..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/configs/mustc_noise.list +++ /dev/null @@ -1,49 +0,0 @@ -"(Applause) NOISE -"(Laughter) VOICE -"(Laughter)" VOICE -(Applause) NOISE -(Applause). NOISE -(Audience) VOICE -(Audio) NOISE -(Beat) NOISE -(Beatboxing) VOICE -(Beep) NOISE -(Beeps) NOISE -(Cheering) VOICE -(Cheers) VOICE -(Claps) NOISE -(Clicking) NOISE -(Clunk) NOISE -(Coughs) NOISE -(Drums) NOISE -(Explosion) NOISE -(Gasps) VOICE -(Guitar) NOISE -(Honk) NOISE -(Laugher) VOICE -(Laughing) VOICE -(Laughs) VOICE -(Laughter) VOICE -(Laughter). VOICE -(Laughter)... VOICE -(Mumbling) VOICE -(Music) NOISE -(Noise) NOISE -(Recording) VOICE -(Ringing) NOISE -(Shouts) VOICE -(Sigh) VOICE -(Sighs) VOICE -(Silence) NOISE -(Singing) VOICE -(Sings) VOICE -(Spanish) VOICE -(Static) NOISE -(Tones) NOISE -(Trumpet) NOISE -(Video) NOISE -(Video): NOISE -(Voice-over) NOISE -(Whistle) NOISE -(Whistling) NOISE -(video): NOISE diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/criterions/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/criterions/__init__.py deleted file mode 100644 index 0d7490139c53e0a8739e68bcbdce5ff622f73191..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/criterions/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import importlib -import os - -for file in os.listdir(os.path.dirname(__file__)): - if file.endswith(".py") and not file.startswith("_"): - criterion_name = file[: file.find(".py")] - importlib.import_module( - "examples.speech_text_joint_to_text.criterions." + criterion_name - ) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/criterions/multi_modality_compound.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/criterions/multi_modality_compound.py deleted file mode 100644 index 292f6fdf487006bd505d8ab0ca23fee852025e39..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/criterions/multi_modality_compound.py +++ /dev/null @@ -1,180 +0,0 @@ -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. -import logging -import math -from dataclasses import dataclass, field - -from fairseq import metrics, utils -from fairseq.criterions import FairseqCriterion, register_criterion -from fairseq.criterions.ctc import CtcCriterion, CtcCriterionConfig -from fairseq.criterions.label_smoothed_cross_entropy import ( - LabelSmoothedCrossEntropyCriterionConfig, -) -from fairseq.logging.meters import safe_round - -from .multi_modality_cross_entropy import SpeechTextPreTrainCrossEntCriterion - -logger = logging.getLogger(__name__) - - -@dataclass -class SpeechTextPreTrainCompoundCriterionConfig( - LabelSmoothedCrossEntropyCriterionConfig -): - zero_infinity: bool = field( - default=False, - metadata={"help": "zero inf loss when source length <= target length"}, - ) - post_process: str = field( - default="none", - metadata={ - "help": "how to post process predictions into words. can be letter, " - "wordpiece, BPE symbols, etc. " - "See fairseq.data.data_utils.post_process() for full list of options" - }, - ) - - -@register_criterion( - "speech_text_pretrain_compound", dataclass=SpeechTextPreTrainCompoundCriterionConfig -) -class SpeechTextPreTrainCompoundCriterion(FairseqCriterion): - def __init__( - self, - task, - sentence_avg, - label_smoothing, - report_accuracy=False, - zero_infinity=False, - post_process=None, - ): - super().__init__(task) - self.xent = SpeechTextPreTrainCrossEntCriterion( - task, sentence_avg, label_smoothing, report_accuracy - ) - cfg_dict = { - "zero_infinity": zero_infinity, - "sentence_avg": sentence_avg, - "post_process": post_process, - } - cfg_ctc = CtcCriterionConfig(**cfg_dict) - self.ctc = CtcCriterion(cfg_ctc, task) - - def forward(self, model, sample, reduce=True): - mode = sample["net_input"]["mode"] - if mode == "sup_speech_ctc": # CTC - sample["net_input"][ - "src_lengths" - ] = None # get downsampled src_lengths from padding_mask - loss, sample_size, logging_output = self.ctc(model, sample, reduce) - logging_output["mode"] = SpeechTextPreTrainCompoundCriterion.mode2value( - "CTC" - ) - else: - loss, sample_size, logging_output = self.xent(model, sample, reduce) - logging_output["mode"] = SpeechTextPreTrainCompoundCriterion.mode2value( - "xent" - ) - - return loss, sample_size, logging_output - - @staticmethod - def logging_outputs_can_be_summed() -> bool: - """ - Whether the logging outputs returned by `forward` can be summed - across workers prior to calling `reduce_metrics`. Setting this - to True will improves distributed training speed. - """ - return True - - @staticmethod - def mode2value(mode): # make the logging_outputs_can_be_summed = True - if mode == "CTC": - return 907 # prime number - if mode == "xent": - return 887 # prime number - return 0 - - @staticmethod - def value2mode(value): - if value % 907 == 0: - return "CTC" - if value % 887 == 0: - return "xent" - raise ValueError("Unknow mode") - - @staticmethod - def reduce_metrics(logging_outputs) -> None: - """Aggregate logging outputs from data parallel training.""" - - def _get_mode(logging_outputs): - mds = [ - SpeechTextPreTrainCompoundCriterion.value2mode(log["mode"]) - for log in logging_outputs - ] - if sum([1 if l != mds[0] else 0 for l in mds]) > 0: - raise ValueError("mode in one mini-batch is expected to be the same!") - return mds[0] - - log_mode = _get_mode(logging_outputs) - if log_mode == "xent": - return SpeechTextPreTrainCrossEntCriterion.reduce_metrics(logging_outputs) - - # ctc loss - loss_sum = utils.item(sum(log.get("loss", 0) for log in logging_outputs)) - ntokens = utils.item(sum(log.get("ntokens", 0) for log in logging_outputs)) - nsentences = utils.item( - sum(log.get("nsentences", 0) for log in logging_outputs) - ) - sample_size = utils.item( - sum(log.get("sample_size", 0) for log in logging_outputs) - ) - - metrics.log_scalar( - "ctc_loss", loss_sum / sample_size / math.log(2), sample_size, round=3 - ) - metrics.log_scalar("ctc_ntokens", ntokens) - metrics.log_scalar("ctc_nsentences", nsentences) - if sample_size != ntokens: - metrics.log_scalar( - "ctc_nll_loss", loss_sum / ntokens / math.log(2), ntokens, round=3 - ) - - c_errors = sum(log.get("c_errors", 0) for log in logging_outputs) - metrics.log_scalar("_c_errors", c_errors) - c_total = sum(log.get("c_total", 0) for log in logging_outputs) - metrics.log_scalar("_c_total", c_total) - w_errors = sum(log.get("w_errors", 0) for log in logging_outputs) - metrics.log_scalar("_w_errors", w_errors) - wv_errors = sum(log.get("wv_errors", 0) for log in logging_outputs) - metrics.log_scalar("_wv_errors", wv_errors) - w_total = sum(log.get("w_total", 0) for log in logging_outputs) - metrics.log_scalar("_w_total", w_total) - - if c_total > 0: - metrics.log_derived( - "uer", - lambda meters: safe_round( - meters["_c_errors"].sum * 100.0 / meters["_c_total"].sum, 3 - ) - if meters["_c_total"].sum > 0 - else float("nan"), - ) - if w_total > 0: - metrics.log_derived( - "wer", - lambda meters: safe_round( - meters["_w_errors"].sum * 100.0 / meters["_w_total"].sum, 3 - ) - if meters["_w_total"].sum > 0 - else float("nan"), - ) - metrics.log_derived( - "raw_wer", - lambda meters: safe_round( - meters["_wv_errors"].sum * 100.0 / meters["_w_total"].sum, 3 - ) - if meters["_w_total"].sum > 0 - else float("nan"), - ) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/criterions/multi_modality_cross_entropy.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/criterions/multi_modality_cross_entropy.py deleted file mode 100644 index b427b303162b3e2c386ee888d673efc959fab637..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/criterions/multi_modality_cross_entropy.py +++ /dev/null @@ -1,100 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. -import torch -from fairseq import utils -from fairseq.criterions import register_criterion -from fairseq.criterions.label_smoothed_cross_entropy import ( - LabelSmoothedCrossEntropyCriterion, - LabelSmoothedCrossEntropyCriterionConfig, - label_smoothed_nll_loss, -) - - -@register_criterion( - "speech_text_pretrain_cross_entropy", - dataclass=LabelSmoothedCrossEntropyCriterionConfig, -) -class SpeechTextPreTrainCrossEntCriterion(LabelSmoothedCrossEntropyCriterion): - def __init__(self, task, sentence_avg, label_smoothing, report_accuracy=False): - super().__init__( - task, sentence_avg, label_smoothing, report_accuracy=report_accuracy - ) - - def forward(self, model, sample, reduce=True): - net_output = model(**sample["net_input"]) - loss, nll_loss, nsentences, ntokens, n_correct = self.compute_loss( - model, net_output, sample, reduce=reduce - ) - sample_size = nsentences if self.sentence_avg else ntokens - logging_output = { - "loss": loss.data, - "nll_loss": nll_loss.data, - "ntokens": ntokens, - "nsentences": nsentences, - "sample_size": sample_size, - } - if self.report_accuracy: - logging_output["n_correct"] = utils.item(n_correct) - logging_output["total"] = utils.item(ntokens) - return loss, sample_size, logging_output - - def get_lprobs_and_target(self, model, net_output, sample): - lprobs = model.get_normalized_probs(net_output, log_probs=True) - target = model.get_targets(sample, net_output) - assert self.ignore_prefix_size == 0 - if self.ignore_prefix_size > 0: - if getattr(lprobs, "batch_first", False): - lprobs = lprobs[:, self.ignore_prefix_size :, :].contiguous() - target = target[:, self.ignore_prefix_size :].contiguous() - else: - lprobs = lprobs[self.ignore_prefix_size :, :, :].contiguous() - target = target[self.ignore_prefix_size :, :].contiguous() - return lprobs, target - - def compute_loss(self, model, net_output, sample, reduce=True): - lprobs, target = self.get_lprobs_and_target(model, net_output, sample) - n_correct = 0 - if isinstance(target, dict): - t_lprobs = target["target_logprobs"] - - if not lprobs.batch_first: - lprobs = lprobs.transpose(0, 1) - t_lprobs = t_lprobs.transpose(0, 1) - nsentences, seq_len = lprobs.size()[:2] - ntokens = nsentences * seq_len - t_probs = t_lprobs.exp() - mask_indices = ( - net_output[1]["mask_indices"][0] - if len(net_output[1]["mask_indices"]) > 0 - else None - ) - - # mask_indices is True for those masking frames - if mask_indices is not None: # B X T - t_probs = t_probs.masked_fill(mask_indices.eq(False).unsqueeze(-1), 0) - ntokens = mask_indices.int().sum() - t_probs = t_probs.detach() - t_lprobs = t_lprobs.detach() - loss = ( - -(t_probs * (lprobs - t_lprobs)).sum() - if reduce - else -(t_probs * (lprobs - t_lprobs)).sum(-1, keepdim=True) - ) - nll_loss = loss - else: - nsentences = target.size(0) - mask = target.ne(self.padding_idx) - loss, nll_loss = label_smoothed_nll_loss( - lprobs.view(-1, lprobs.size(-1)), - target.view(-1), - self.eps, - ignore_index=self.padding_idx, - reduce=reduce, - ) - n_correct = torch.sum( - lprobs.argmax(-1).masked_select(mask).eq(target.masked_select(mask)) - ) - ntokens = torch.sum(mask) - return loss, nll_loss, nsentences, ntokens, n_correct diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/criterions/text_guide_cross_entropy_acc.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/criterions/text_guide_cross_entropy_acc.py deleted file mode 100644 index 8ada7aa630ead93dd1ec466fb16332b7c77e1248..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/criterions/text_guide_cross_entropy_acc.py +++ /dev/null @@ -1,223 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. -import math - -import torch -import torch.nn.functional as F -from fairseq import metrics, utils -from fairseq.criterions import FairseqCriterion, register_criterion -from fairseq.criterions.label_smoothed_cross_entropy import label_smoothed_nll_loss - - -@register_criterion("guided_label_smoothed_cross_entropy_with_accuracy") -class GuidedCrossEntAccCriterion(FairseqCriterion): - def __init__( - self, - task, - sentence_avg, - guide_alpha, - text_input_cost_ratio, - label_smoothing, - disable_text_guide_update_num=0, - attentive_cost_regularization=0, - ): - """ - guide_alpha: alpha to inteplate nll and kd loss - text_input_cost_ratio: loss ratio for text only input data - label_smoothing: label smoothing ratio - disable_text_guide_update_num: only use nll loss for the first N updates - attentive_cost_regularization: ratio fo attentive cost - """ - super().__init__(task) - self.alpha = guide_alpha - self.attn_beta = attentive_cost_regularization - self.sentence_avg = sentence_avg - self.eps = label_smoothing - self.text_input_cost_ratio = text_input_cost_ratio - self.disable_update_num = disable_text_guide_update_num - assert self.alpha >= 0 and self.alpha <= 1.0 - - @staticmethod - def add_args(parser): - """Add criterion-specific arguments to the parser.""" - # fmt: off - parser.add_argument('--label-smoothing', default=0., type=float, metavar='D', - help='epsilon for label smoothing, 0 means no label smoothing') - # fmt: off - parser.add_argument('--guide-alpha', default=0., type=float, metavar='D', - help='alpha to merge kd cost from text to speech input with ce loss') - # fmt: off - parser.add_argument('--disable-text-guide-update-num', default=0, type=int, metavar='D', - help='disable guided target from text for the first N updates.') - parser.add_argument("--attentive-cost-regularization", default=0.0, type=float, metavar='D', - help="use encoder attentive loss regularization with cost ratio D") - parser.add_argument("--attentive-cost-without-normalize", action='store_true', - help="Don't do normalization during attentive cost computation") - - def forward(self, model, sample, reduce=True): - reduction = 'sum' if reduce else 'none' - net_input = sample["net_input"] - net_output = model(**net_input) - attn_cost = None - lprobs = model.get_normalized_probs(net_output, log_probs=True) - is_dual_input = True if net_input['src_tokens'] is not None and net_input.get('src_txt_tokens') is not None else False - target = model.get_targets(sample, net_output) - src_token_num = 0 - if is_dual_input: - # lprobs_spch from speech encoder and lprobs_text from text encoder - lprobs_spch, lprobs_text = torch.chunk(lprobs, 2) - lprobs_spch.batch_first = lprobs.batch_first - lprobs_text.batch_first = lprobs.batch_first - - speech_loss, speech_nll_loss, speech_correct, speech_total = \ - self.guide_loss_and_acc(model, lprobs_spch, lprobs_text, target, reduce=(reduction == 'sum')) - text_loss, text_nll_loss, text_correct, text_total = self.compute_loss_and_acc(model, lprobs_text, target, reduction=reduction) - loss = (speech_loss + text_loss) - nll_loss = (speech_nll_loss + text_nll_loss) - correct = speech_correct + text_correct - total = speech_total + text_total - - attn_cost = net_output[1].get('attn_cost') - if attn_cost is not None: - # attn_cost is batch_first and padding tokens have been masked already - src_token_num = attn_cost.ne(0).sum() - attn_cost = attn_cost.sum() - loss = loss + attn_cost * self.attn_beta - else: - attn_cost = 0 - else: - loss, nll_loss, correct, total = self.compute_loss_and_acc(model, lprobs, target, reduction=reduction) - if sample["net_input"]['src_tokens'] is None: # text input only - loss = loss * self.text_input_cost_ratio - speech_loss = None - speech_nll_loss = None - - sample_size, logging_output = self.get_logging_output( - sample, loss, nll_loss, correct, total, src_token_num, speech_loss, speech_nll_loss, attn_cost, is_dual_input - ) - return loss, sample_size, logging_output - - def compute_loss_and_acc(self, model, lprobs, target, reduction='sum'): - if not lprobs.batch_first: - lprobs = lprobs.transpose(0, 1) - lprobs = lprobs.view(-1, lprobs.size(-1)) # -> (B x T) x C - target = target.view(-1) - loss, nll_loss = label_smoothed_nll_loss( - lprobs, target, self.eps, ignore_index=self.padding_idx, reduce=(reduction == 'sum'), - ) - - mask = target.ne(self.padding_idx) - correct = torch.sum(lprobs.argmax(1).masked_select(mask).eq(target.masked_select(mask))) - total = torch.sum(mask) - return loss, nll_loss, correct, total - - def guide_loss_and_acc(self, model, lprobs, lprobs_teacher, target, reduce=True): - """ lprobs_teacher is used as guide for lprobs """ - if self.alpha == 0.0 or model.num_updates < self.disable_update_num: - return self.compute_loss_and_acc(model, lprobs, target, reduction=('sum' if reduce else 'none')) - if not lprobs.batch_first: - lprobs = lprobs.transpose(0, 1) - lprobs_teacher = lprobs_teacher.transpose(0, 1) - - lprobs = lprobs.view(-1, lprobs.size(-1)).float() # -> (B x T) x C - lprobs_teacher = lprobs_teacher.view(-1, lprobs_teacher.size(-1)).float() # -> (B x T) x C - target = target.view(-1) - loss = F.nll_loss(lprobs, target, ignore_index=self.padding_idx, reduction='sum' if reduce else 'none') - nll_loss = loss - probs_teacher = lprobs_teacher.exp().masked_fill_(target.unsqueeze(-1).eq(self.padding_idx), 0) - probs_teacher = probs_teacher.detach() - guide_loss = -(probs_teacher*lprobs).sum() if reduce else -(probs_teacher*lprobs).sum(-1, keepdim=True) - loss = self.alpha*guide_loss + (1.0 - self.alpha)*loss - - mask = target.ne(self.padding_idx) - correct = torch.sum(lprobs.argmax(1).masked_select(mask).eq(target.masked_select(mask))) - total = torch.sum(mask) - return loss, nll_loss, correct, total - - def get_logging_output( - self, - sample, - loss, - nll_loss, - correct, - total, - src_token_num=0, - speech_loss=None, - speech_nll_loss=None, - attn_cost=None, - is_dual_input=False, - ): - - sample_size = ( - sample["target"].size(0) if self.sentence_avg else sample["ntokens"] - ) - mul_size = 2 if is_dual_input else 1 - - logging_output = { - "loss": utils.item(loss.data), # * sample['ntokens'], - "nll_loss": utils.item(nll_loss.data), # * sample['ntokens'], - "ntokens": sample["ntokens"]*mul_size, - "nsentences": sample["target"].size(0)*mul_size, - "sample_size": sample_size*mul_size, - "correct": utils.item(correct.data), - "total": utils.item(total.data), - "src_token_num": utils.item(src_token_num.data) if src_token_num > 0 else 0, - "nframes": torch.sum(sample["net_input"]["src_lengths"]).item(), - } - - if speech_loss is not None: - logging_output["speech_loss"] = utils.item(speech_loss.data) - logging_output["speech_nll_loss"] = utils.item(speech_nll_loss.data) - logging_output["sample_size_speech_cost"] = sample_size - logging_output["speech_attn_loss"] = attn_cost - - return sample_size*mul_size, logging_output - - @staticmethod - def aggregate_logging_outputs(logging_outputs): - """Aggregate logging outputs from data parallel training.""" - correct_sum = sum(log.get("correct", 0) for log in logging_outputs) - total_sum = sum(log.get("total", 0) for log in logging_outputs) - src_token_sum = sum(log.get("src_token_num", 0) for log in logging_outputs) - loss_sum = sum(log.get("loss", 0) for log in logging_outputs) - nll_loss_sum = sum(log.get("nll_loss", 0) for log in logging_outputs) - ntokens = sum(log.get("ntokens", 0) for log in logging_outputs) - nsentences = sum(log.get("nsentences", 0) for log in logging_outputs) - sample_size = sum(log.get("sample_size", 0) for log in logging_outputs) - nframes = sum(log.get("nframes", 0) for log in logging_outputs) - speech_loss_sum = sum(log.get("speech_loss", 0) for log in logging_outputs) - speech_nll_loss_sum = sum(log.get("speech_nll_loss", 0) for log in logging_outputs) - speech_attn_loss_sum = sum(log.get("speech_attn_loss", 0) for log in logging_outputs) - sample_size_speech = sum(log.get("sample_size_speech_cost", 0) for log in logging_outputs) - - agg_output = { - "loss": loss_sum / sample_size / math.log(2) if sample_size > 0 else 0.0, - "nll_loss": nll_loss_sum / sample_size / math.log(2) if sample_size > 0 else 0.0, - # if args.sentence_avg, then sample_size is nsentences, and loss - # is per-sentence loss; else sample_size is ntokens, and the loss - # becomes per-output token loss - "speech_loss": speech_loss_sum / sample_size_speech / math.log(2) if sample_size_speech > 0 else 0.0, - "speech_nll_loss": speech_nll_loss_sum / sample_size_speech / math.log(2) if sample_size_speech > 0 else 0.0, - "speech_attn_loss": speech_attn_loss_sum / src_token_sum / math.log(2) if src_token_sum > 0 else 0.0, - "ntokens": ntokens, - "nsentences": nsentences, - "nframes": nframes, - "sample_size": sample_size, - "acc": correct_sum * 100.0 / total_sum if total_sum > 0 else 0.0, - "correct": correct_sum, - "total": total_sum, - "src_token_num": src_token_sum, - # total is the number of validate tokens - } - return agg_output - - @classmethod - def reduce_metrics(cls, logging_outputs): - """Aggregate logging outputs from data parallel training.""" - agg_logging_outputs = cls.aggregate_logging_outputs(logging_outputs) - for k, v in agg_logging_outputs.items(): - if k in {'nsentences', 'ntokens', 'sample_size'}: - continue - metrics.log_scalar(k, v, round=3) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/data/pair_denoising_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/data/pair_denoising_dataset.py deleted file mode 100644 index 2456abfbc011fe014c0ee66001deb92f3f40ca1f..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/data/pair_denoising_dataset.py +++ /dev/null @@ -1,317 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import copy -import math -import re - -import torch -from fairseq.data import data_utils -from fairseq.data.language_pair_dataset import LanguagePairDataset - - -# Part of the code is modified from DenoisingDataset -# compared with DenoisingDataset, no permute_sentences or documents (rotate_ratio, permute_sentence_ratio) -class LanguagePairDenoisingDataset(LanguagePairDataset): - def __init__( - self, - src, - src_sizes, - src_dict, - tgt, - tgt_sizes, - tgt_dict, - mask_idx, - mask_whole_words, - seed, - args, - left_pad_source=True, - left_pad_target=False, - shuffle=True, - input_feeding=True, - remove_eos_from_source=False, - append_eos_to_target=False, - align_dataset=None, - constraints=None, - append_bos=False, - eos=None, - num_buckets=0, - src_lang_id=None, - tgt_lang_id=None, - pad_to_multiple=1, - ): - super().__init__( - src, - src_sizes, - src_dict, - tgt, - tgt_sizes, - tgt_dict, - left_pad_source, - left_pad_target, - shuffle, - input_feeding, - remove_eos_from_source, - append_eos_to_target, - align_dataset, - constraints, - append_bos, - eos, - num_buckets, - src_lang_id, - tgt_lang_id, - pad_to_multiple, - ) - - self.mask_idx = mask_idx - self.mask_whole_word = mask_whole_words - self.mask_ratio = args.mask - self.random_ratio = args.mask_random - self.insert_ratio = args.insert - - self.replace_length = args.replace_length - - if self.replace_length not in [-1, 0, 1]: - raise ValueError(f"invalid arg: replace_length={self.replace_length}") - if args.mask_length not in ["subword", "word", "span-poisson"]: - raise ValueError(f"invalid arg: mask-length={args.mask_length}") - if args.mask_length == "subword" and args.replace_length not in [0, 1]: - raise ValueError("if using subwords, use replace-length=1 or 0") - - self.mask_span_distribution = None - if args.mask_length == "span-poisson": - # Text infilling: "A number of text spans are sampled, with span lengths drawn from a Poisson distribution (λ = 3). Each span is replaced with a single [MASK] token. 0-length spans correspond to the insertion of [MASK] tokens." - _lambda = args.poisson_lambda - - lambda_to_the_k = 1 - e_to_the_minus_lambda = math.exp(-_lambda) - k_factorial = 1 - ps = [] - for k in range(0, 128): - ps.append(e_to_the_minus_lambda * lambda_to_the_k / k_factorial) - lambda_to_the_k *= _lambda - k_factorial *= k + 1 - if ps[-1] < 0.0000001: - break - ps = torch.FloatTensor(ps) - self.mask_span_distribution = torch.distributions.Categorical(ps) - - self.epoch = 0 - self.seed = seed - - def _is_phoneme(x): - if re.search("", - "", - "", - "", - "", - "", - ): - return False - return True - - self.voc_valid_ids = torch.LongTensor( - [i for i, x in enumerate(self.src_dict.symbols) if _is_phoneme(x)] - ) - self.voc_valid_size = self.voc_valid_ids.size(0) - - @property - def can_reuse_epoch_itr_across_epochs(self): - return False - - def set_epoch(self, epoch, **unused): - self.epoch = epoch - - def __getitem__(self, index): - tgt_item = self.tgt[index] if self.tgt is not None else None - src_item = copy.deepcopy(self.src[index]) - with data_utils.numpy_seed(self.seed, self.epoch, index): - source = src_item - assert source[-1] == self.eos - if self.mask_ratio > 0: - source = self.add_whole_word_mask(source, self.mask_ratio) - - if self.insert_ratio > 0: - source = self.add_insertion_noise(source, self.insert_ratio) - src_item = source - - if self.append_eos_to_target: - eos = self.tgt_dict.eos() if self.tgt_dict else self.src_dict.eos() - if self.tgt and self.tgt[index][-1] != eos: - tgt_item = torch.cat([self.tgt[index], torch.LongTensor([eos])]) - - if self.append_bos: - bos = self.tgt_dict.bos() if self.tgt_dict else self.src_dict.bos() - if self.tgt and self.tgt[index][0] != bos: - tgt_item = torch.cat([torch.LongTensor([bos]), self.tgt[index]]) - - bos = self.src_dict.bos() - if src_item[0] != bos: - src_item = torch.cat([torch.LongTensor([bos]), src_item]) - - if self.remove_eos_from_source: - eos = self.src_dict.eos() - if src_item[-1] == eos: - src_item = src_item[:-1] - - example = { - "id": index, - "source": src_item, - "target": tgt_item, - } - if self.align_dataset is not None: - example["alignment"] = self.align_dataset[index] - if self.constraints is not None: - example["constraints"] = self.constraints[index] - if self.src_lang_id is not None: - example["src_lang_id"] = self.src_lang_id - if self.tgt_lang_id is not None: - example["tgt_lang_id"] = self.tgt_lang_id - return example - - # following functions are borrowed from denoising_dataset - def word_starts(self, source): - if self.mask_whole_word is not None: - is_word_start = self.mask_whole_word.gather(0, source) - else: - is_word_start = torch.ones(source.size()) - is_word_start[0] = 0 - is_word_start[-1] = 0 - return is_word_start - - def add_whole_word_mask(self, source, p): - is_word_start = self.word_starts(source) - num_to_mask = int(math.ceil(is_word_start.float().sum() * p)) - num_inserts = 0 - if num_to_mask == 0: - return source - - if self.mask_span_distribution is not None: - lengths = self.mask_span_distribution.sample(sample_shape=(num_to_mask,)) - - # Make sure we have enough to mask - cum_length = torch.cumsum(lengths, 0) - while cum_length[-1] < num_to_mask: - lengths = torch.cat( - [ - lengths, - self.mask_span_distribution.sample(sample_shape=(num_to_mask,)), - ], - dim=0, - ) - cum_length = torch.cumsum(lengths, 0) - - # Trim to masking budget - i = 0 - while cum_length[i] < num_to_mask: - i += 1 - lengths[i] = num_to_mask - (0 if i == 0 else cum_length[i - 1]) - num_to_mask = i + 1 - lengths = lengths[:num_to_mask] - - # Handle 0-length mask (inserts) separately - lengths = lengths[lengths > 0] - num_inserts = num_to_mask - lengths.size(0) - num_to_mask -= num_inserts - if num_to_mask == 0: - return self.add_insertion_noise(source, num_inserts / source.size(0)) - - assert (lengths > 0).all() - else: - lengths = torch.ones((num_to_mask,)).long() - assert is_word_start[-1] == 0 - word_starts = is_word_start.nonzero(as_tuple=False) - indices = word_starts[ - torch.randperm(word_starts.size(0))[:num_to_mask] - ].squeeze(1) - mask_random = torch.FloatTensor(num_to_mask).uniform_() < self.random_ratio - - source_length = source.size(0) - assert source_length - 1 not in indices - to_keep = torch.ones(source_length, dtype=torch.bool) - is_word_start[ - -1 - ] = 255 # acts as a long length, so spans don't go over the end of doc - if self.replace_length == 0: - to_keep[indices] = 0 - else: - # keep index, but replace it with [MASK] - source[indices] = self.mask_idx - source[indices[mask_random]] = self.voc_valid_ids[ - torch.randint(0, self.voc_valid_size - 1, size=(mask_random.sum(),)) - ] - - if self.mask_span_distribution is not None: - assert len(lengths.size()) == 1 - assert lengths.size() == indices.size() - lengths -= 1 - while indices.size(0) > 0: - assert lengths.size() == indices.size() - lengths -= is_word_start[indices + 1].long() - uncompleted = lengths >= 0 - indices = indices[uncompleted] + 1 - mask_random = mask_random[uncompleted] - lengths = lengths[uncompleted] - if self.replace_length != -1: - # delete token - to_keep[indices] = 0 - else: - # keep index, but replace it with [MASK] - source[indices] = self.mask_idx - source[indices[mask_random]] = self.voc_valid_ids[ - torch.randint( - 0, self.voc_valid_size - 1, size=(mask_random.sum(),) - ) - ] - else: - # A bit faster when all lengths are 1 - while indices.size(0) > 0: - uncompleted = is_word_start[indices + 1] == 0 - indices = indices[uncompleted] + 1 - mask_random = mask_random[uncompleted] - if self.replace_length != -1: - # delete token - to_keep[indices] = 0 - else: - # keep index, but replace it with [MASK] - source[indices] = self.mask_idx - source[indices[mask_random]] = self.voc_valid_ids[ - torch.randint( - 0, self.voc_valid_size - 1, size=(mask_random.sum(),) - ) - ] - - assert source_length - 1 not in indices - - source = source[to_keep] - - if num_inserts > 0: - source = self.add_insertion_noise(source, num_inserts / source.size(0)) - - return source - - def add_insertion_noise(self, tokens, p): - if p == 0.0: - return tokens - - num_tokens = len(tokens) - n = int(math.ceil(num_tokens * p)) - - noise_indices = torch.randperm(num_tokens + n - 2)[:n] + 1 - noise_mask = torch.zeros(size=(num_tokens + n,), dtype=torch.bool) - noise_mask[noise_indices] = 1 - result = torch.LongTensor(n + len(tokens)).fill_(-1) - - num_random = int(math.ceil(n * self.random_ratio)) - result[noise_indices[num_random:]] = self.mask_idx - result[noise_indices[:num_random]] = self.voc_valid_ids[ - torch.randint(0, self.voc_valid_size - 1, size=(num_random,)) - ] - - result[~noise_mask] = tokens - - assert (result >= 0).all() - return result diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/docs/ende-mustc.md b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/docs/ende-mustc.md deleted file mode 100644 index 1acf6e001be3b4ddfe40b3d4dc25bb4754a97ef3..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/docs/ende-mustc.md +++ /dev/null @@ -1,118 +0,0 @@ -[[Back]](..) - -# Joint Speech Text Training for the MuST-C English to German Speech Translation task - -Joint Training Baseline: it is based on paper ["A general multi-task learning framework to leverage text data for speech to text tasks"](https://arxiv.org/pdf/2010.11338.pdf) - -Enhanced Joint Training: the joint training is enhanced with pre-trained models, cross attentive regularization and online knowledge distillation based on paper ["Improving Speech Translation by Understanding and Learning from the Auxiliary Text Translation Task"](https://research.fb.com/publications/improving-speech-translation-by-understanding-and-learning-from-the-auxiliary-text-translation-task) - -## Prepare Data -#### Download files -- Sentence piece model [spm.model](https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/must_c/en_de/spm.model) -- Dictionary [dict.txt](https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/must_c/en_de/dict.txt) -- config [config.yaml](https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/must_c/en_de/config.yaml) -#### Prepare MuST-C data set -- Please follow the data preparation in the [S2T example](https://github.com/pytorch/fairseq/blob/main/examples/speech_to_text/docs/mustc_example.md) -- Convert source text under the "src_text" column in the tsv file into phoneme representation. -```bash - python examples/speech_text_joint_to_text/scripts/g2p_encode.py \ - --lower-case --do-filter --use-word-start --no-punc \ - --reserve-word examples/speech_text_joint_to_text/configs/mustc_noise.list \ - --data-path ${must_c_en_de_src_text} \ - --out-path ${must_c_en_de_src_text_pho} -``` -- Replace the source text under the "src_text" column in the tsv file with the corresponding phoneme reprentation generated in the step above. -Below is the snapshot for the MuST-C en-de dev tsv -``` -id audio n_frames tgt_text src_text speaker -ted_767_0 en-de/flac.zip:10071514743:48445 56160 Heute spreche ich zu Ihnen über Energie und Klima. ▁AY1 M ▁G OW1 IH0 NG ▁T UW1 ▁T AO1 K ▁T AH0 D EY1 ▁AH0 B AW1 T ▁EH1 N ER0 JH IY0 ▁AH0 N D ▁K L AY1 M AH0 T spk.767_ -ted_767_1 en-de/flac.zip:1214217978:205678 226080 Und das überrascht vielleicht etwas, weil sich meine Vollzeitbeschäftigung bei der Stiftung hauptsächlich um Impfstoffe und Saatgut dreht, um die Dinge, die wir erfinden und liefern müssen um den ärmsten 2 Milliarden ein besseres Leben zu ermöglichen. ▁AH0 N D ▁DH AE1 T ▁M AY1 T ▁S IY1 M ▁AH0 ▁B IH1 T ▁S ER0 P R AY1 Z IH0 NG ▁B IH0 K AO1 Z ▁M AY1 ▁F UH1 L ▁T AY1 M ▁W ER1 K ▁AE1 T ▁DH AH0 ▁F AW0 N D EY1 SH AH0 N ▁IH1 Z ▁M OW1 S T L IY0 ▁AH0 B AW1 T ▁V AE2 K S IY1 N Z ▁AH0 N D ▁S IY1 D Z ▁AH0 B AW1 T ▁DH AH0 ▁TH IH1 NG Z ▁DH AE1 T ▁W IY1 ▁N IY1 D ▁T UW1 ▁IH0 N V EH1 N T ▁AH0 N D ▁D IH0 L IH1 V ER0 ▁T UW1 ▁HH EH1 L P ▁DH AH0 ▁P UH1 R IH0 S T ▁T UW1 ▁B IH1 L Y AH0 N ▁L AY1 V ▁B EH1 T ER0 ▁L IH1 V Z spk.767_ -``` -- Prepare phoneme dictionary and save to $MANIFEST_ROOT as [src_dict.txt](https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/must_c/en_de/src_dict.txt) -#### Prepare WMT text data -- [Download wmt data](https://github.com/pytorch/fairseq/blob/main/examples/translation/prepare-wmt14en2de.sh) -- Convert source text (English) into phoneme representation as above -- Generate binary parallel files with "fairseq-preprocess" from fairseq for training and validation. The source input is English phoneme representation and the target input is German sentencepiece token . The output is saved under $parallel_text_data - -## Training -The model is trained with 8 v100 GPUs. - -#### Download pretrained models -- [pretrain_encoder](https://dl.fbaipublicfiles.com/fairseq/s2t/mustc_joint_asr_transformer_m.pt) -- [pretrain_nmt](https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/must_c/en_de/checkpoint_mt.pt) - -#### Training scripts -- Jointly trained model from scratch -```bash -python train.py ${MANIFEST_ROOT} \ - --save-dir ${save_dir} \ - --num-workers 8 \ - --task speech_text_joint_to_text \ - --arch dualinputs2ttransformer_s \ - --user-dir examples/speech_text_joint_to_text \ - --max-epoch 100 --update-mix-data \ - --optimizer adam --lr-scheduler inverse_sqrt \ - --lr 0.001 --update-freq 4 --clip-norm 10.0 \ - --criterion guided_label_smoothed_cross_entropy_with_accuracy \ - --label-smoothing 0.1 --max-tokens 10000 --max-tokens-text 10000 \ - --max-positions-text 400 --seed 2 --speech-encoder-layers 12 \ - --text-encoder-layers 6 --encoder-shared-layers 6 --decoder-layers 6 \ - --dropout 0.1 --warmup-updates 20000 \ - --text-sample-ratio 0.25 --parallel-text-data ${parallel_text_data} \ - --text-input-cost-ratio 0.5 --enc-grad-mult 2.0 --add-speech-eos \ - --log-format json --langpairs en-de --noise-token '"'"'▁NOISE'"'"' \ - --mask-text-ratio 0.0 --max-tokens-valid 20000 --ddp-backend no_c10d \ - --log-interval 100 --data-buffer-size 50 --config-yaml config.yaml \ - --keep-last-epochs 10 -``` -- Jointly trained model with good initialization, cross attentive loss and online knowledge distillation -```bash -python train.py ${MANIFEST_ROOT} \ - --save-dir ${save_dir} \ - --num-workers 8 \ - --task speech_text_joint_to_text \ - --arch dualinputs2ttransformer_m \ - --user-dir examples/speech_text_joint_to_text \ - --max-epoch 100 --update-mix-data \ - --optimizer adam --lr-scheduler inverse_sqrt \ - --lr 0.002 --update-freq 4 --clip-norm 10.0 \ - --criterion guided_label_smoothed_cross_entropy_with_accuracy \ - --guide-alpha 0.8 --disable-text-guide-update-num 5000 \ - --label-smoothing 0.1 --max-tokens 10000 --max-tokens-text 10000 \ - --max-positions-text 400 --seed 2 --speech-encoder-layers 12 \ - --text-encoder-layers 6 --encoder-shared-layers 6 --decoder-layers 6 \ - --dropout 0.1 --warmup-updates 20000 --attentive-cost-regularization 0.02 \ - --text-sample-ratio 0.25 --parallel-text-data ${parallel_text_data} \ - --text-input-cost-ratio 0.5 --enc-grad-mult 2.0 --add-speech-eos \ - --log-format json --langpairs en-de --noise-token '"'"'▁NOISE'"'"' \ - --mask-text-ratio 0.0 --max-tokens-valid 20000 --ddp-backend no_c10d \ - --log-interval 100 --data-buffer-size 50 --config-yaml config.yaml \ - --load-pretrain-speech-encoder ${pretrain_encoder} \ - --load-pretrain-decoder ${pretrain_nmt} \ - --load-pretrain-text-encoder-last ${pretrain_nmt} \ - --keep-last-epochs 10 -``` - -## Evaluation -```bash -python ./fairseq_cli/generate.py \ - ${MANIFEST_ROOT} \ - --task speech_text_joint_to_text \ - --max-tokens 25000 \ - --nbest 1 \ - --results-path ${infer_results} \ - --batch-size 512 \ - --path ${model} \ - --gen-subset tst-COMMON_st \ - --config-yaml config.yaml \ - --scoring sacrebleu \ - --beam 5 --lenpen 1.0 \ - --user-dir examples/speech_text_joint_to_text \ - --load-speech-only -``` - -## Results (Joint training with initialization + CAR + online KD) -|Direction|En-De | En-Es | En-Fr | -|---|---|---|---| -|BLEU|27.4| 31.2 | 37.6 | -|checkpoint | [link](https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/must_c/en_de/checkpoint_ave_10.pt) |[link](https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/must_c/en_es/checkpoint_ave_10.pt)|[link](https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/must_c/en_fr/checkpoint_ave_10.pt)| diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/docs/iwslt2021.md b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/docs/iwslt2021.md deleted file mode 100644 index 0af0fbff1bd4d113f910912ce2dad516ff6eea79..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/docs/iwslt2021.md +++ /dev/null @@ -1,76 +0,0 @@ -[[Back]](..) - -# Joint Speech Text Training for the 2021 IWSLT multilingual speech translation - -This directory contains the code from paper ["FST: the FAIR Speech Translation System for the IWSLT21 Multilingual Shared Task"](https://arxiv.org/pdf/2107.06959.pdf). - -## Prepare Data -#### Download files -- Sentence piece model [spm.model](https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/iwslt/iwslt_data/spm.model) -- Dictionary [tgt_dict.txt](https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/iwslt/iwslt_data/dict.txt) -- Config [config.yaml](https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/iwslt/iwslt_data/config.yaml) - -#### Prepare -- Please follow the data preparation in [speech-to-text](https://github.com/pytorch/fairseq/blob/main/examples/speech_to_text/docs/mtedx_example.md) with option "--use-audio-input" for raw audio tsv files. -- Prepare tsv files with phoneme based source text (under column 'src_text') as [MuST-C](ende-mustc.md) example. - - -## Training - -#### Download pretrained models -- [Pretrained mbart model](https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/iwslt/iwslt_data/mbart.pt) -- [Pretrained w2v model](https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/iwslt/iwslt_data/xlsr_53_56k.pt) - - -#### Training scripts - -```bash -python train.py ${MANIFEST_ROOT} \ - --save-dir ${save_dir} \ - --user-dir examples/speech_text_joint_to_text \ - --train-subset train_es_en_tedx,train_es_es_tedx,train_fr_en_tedx,train_fr_es_tedx,train_fr_fr_tedx,train_it_it_tedx,train_pt_en_tedx,train_pt_pt_tedx \ - --valid-subset valid_es_en_tedx,valid_es_es_tedx,valid_es_fr_tedx,valid_es_it_tedx,valid_es_pt_tedx,valid_fr_en_tedx,valid_fr_es_tedx,valid_fr_fr_tedx,valid_fr_pt_tedx,valid_it_en_tedx,valid_it_es_tedx,valid_it_it_tedx,valid_pt_en_tedx,valid_pt_es_tedx,valid_pt_pt_tedx \ - --config-yaml config.yaml --ddp-backend no_c10d \ - --num-workers 2 --task speech_text_joint_to_text \ - --criterion guided_label_smoothed_cross_entropy_with_accuracy \ - --label-smoothing 0.3 --guide-alpha 0.8 \ - --disable-text-guide-update-num 5000 --arch dualinputxmtransformer_base \ - --max-tokens 500000 --max-sentences 3 --max-tokens-valid 800000 \ - --max-source-positions 800000 --enc-grad-mult 2.0 \ - --attentive-cost-regularization 0.02 --optimizer adam \ - --clip-norm 1.0 --log-format simple --log-interval 200 \ - --keep-last-epochs 5 --seed 1 \ - --w2v-path ${w2v_path} \ - --load-pretrained-mbart-from ${mbart_path} \ - --max-update 1000000 --update-freq 4 \ - --skip-invalid-size-inputs-valid-test \ - --skip-encoder-projection --save-interval 1 \ - --attention-dropout 0.3 --mbart-dropout 0.3 \ - --finetune-w2v-params all --finetune-mbart-decoder-params all \ - --finetune-mbart-encoder-params all --stack-w2v-mbart-encoder \ - --drop-w2v-layers 12 --normalize \ - --lr 5e-05 --lr-scheduler inverse_sqrt --warmup-updates 5000 -``` - -## Evaluation -```bash -python ./fairseq_cli/generate.py - ${MANIFEST_ROOT} \ - --task speech_text_joint_to_text \ - --user-dir ./examples/speech_text_joint_to_text \ - --load-speech-only --gen-subset test_es_en_tedx \ - --path ${model} \ - --max-source-positions 800000 \ - --skip-invalid-size-inputs-valid-test \ - --config-yaml config.yaml \ - --infer-target-lang en \ - --max-tokens 800000 \ - --beam 5 \ - --results-path ${RESULTS_DIR} \ - --scoring sacrebleu -``` -The trained model can be downloaded [here](https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/iwslt/iwslt_data/checkpoint17.pt) - -|direction|es_en|fr_en|pt_en|it_en|fr_es|pt_es|it_es|es_es|fr_fr|pt_pt|it_it| -|---|---|---|---|---|---|---|---|---|---|---|---| -|BLEU|31.62|36.93|35.07|27.12|38.87|35.57|34.13|74.59|74.64|70.84|69.76| diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/docs/pre-training.md b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/docs/pre-training.md deleted file mode 100644 index 6d9e2cb0bbc5afdd3f31e9e6867ec435f4ccf6cc..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/docs/pre-training.md +++ /dev/null @@ -1,192 +0,0 @@ -[[Back]](..) - -# Unified Speech-Text Pre-training for Speech Translation and Recognition - -This directory contains the pre-training recipes from paper ["Unified Speech-Text Pre-training for Speech Translation and Recognition"](https://arxiv.org/abs/2204.05409). - -## Librispeech ASR Pre-training -### Prepare Data -#### Download files -#### Prepare pre-training data -- Text to text task (T2T): prepare the binary data following the similar steps in [EN_DE Joint training](./ende-mustc.md). The source data is presented as phomeme token sequence and the target data is coded as subword tokens via SentencePiece. The text data is downloaded from [openslr](https://www.openslr.org/12) -- Self-supervised speech learning task (SSL): The data is prepared as [wav2vec 2.0](https://github.com/pytorch/fairseq/tree/main/examples/wav2vec/README.md) -- Speech to phoneme classification task (S2P): The tsv file contains 5 fields: "id", "audio", "n_frames", "tgt_text", and "align". The tgt_text field is corresponding to the phoneme based representation of the speech data. "align" field contains the alignment information. The phoneme level forced alignment for the labelled speech data (i.e. Librispeech) can be obtained via [kaldi](http://kaldi-asr.org) or [MFA](https://montrealcorpustools.github.io/Montreal-Forced-Aligner/). The segmentation information is normalized to 0$\sim$1 for the whole utterance. The snapshot of the tsv file is below: -``` -id audio n_frames tgt_text align -116-288045-0000 /librispeech/dev-other/116/288045/116-288045-0000.flac 170400 ▁AE1 Z AY1 ▁AH0 P R OW1 CH T ▁DH AH1 ▁S IH1 T IY0 AY1 ▁HH ER1 D ▁B EH1 L Z ▁R IH1 NG IH0 NG ▁AE1 N D AH0 ▁L IH1 T AH0 L ▁L EY1 T ER0 AY1 ▁F AW1 N D ▁DH AH0 ▁S T R IY1 T S ▁AH0 S T IH1 R ▁W IH0 TH ▁TH R AO1 NG Z ▁AH0 V ▁W EH1 L ▁D R EH1 S T ▁P IY1 P AH0 L ▁IH1 N ▁F AE1 M L IY0 ▁G R UW1 P S ▁W EH1 N D IH0 NG ▁DH EH1 R ▁W EY1 ▁HH IH1 DH ER0 ▁AH0 N D ▁TH IH1 DH ER0 0.047977 0.056444 0.064911 0.075259 0.081844 0.089370 0.095014 0.104421 0.109125 0.111947 0.115710 0.120414 0.134525 0.141110 0.143932 0.174036 0.176858 0.190028 0.199436 0.207902 0.218250 0.224835 0.231421 0.242709 0.251176 0.257761 0.263405 0.268109 0.270931 0.290687 0.342427 0.349953 0.353716 0.356538 0.360301 0.363123 0.365945 0.368768 0.371590 0.376294 0.384760 0.394167 0.401693 0.409219 0.419567 0.430856 0.441204 0.444026 0.446849 0.449671 0.456256 0.463782 0.471308 0.477893 0.486359 0.491063 0.494826 0.501411 0.512700 0.517404 0.520226 0.534337 0.540922 0.545626 0.550329 0.559737 0.568203 0.583255 0.592662 0.600188 0.603951 0.611477 0.619003 0.624647 0.634055 0.639699 0.646284 0.653810 0.659454 0.664158 0.670743 0.682032 0.687676 0.692380 0.708373 0.713076 0.719661 0.729069 0.740357 0.744120 0.748824 0.752587 0.761994 0.770461 0.781750 0.790216 0.805268 0.808090 0.823142 0.832549 0.836312 0.840075 0.843838 0.851364 0.854186 0.857008 0.862653 0.878645 0.898401 0.901223 0.906867 0.913452 0.920038 0.926623 0.934149 0.939793 0.942615 0.945437 0.952023 0.957667 0.977422 1.000000 - -``` -- Speech to text task (S2T): The data preparation follow the steps in [EN_DE Joint training](./ende-mustc.md). - -#### Prepare fine-tuning data: -We re-use the data from T2T and S2T tasks in the fine-tuning stage. - -### Model Build -#### Pre-training -``` -python train.py $T2T_DATA \ - --save-dir $SAVE_PRE_PATH --user-dir examples/speech_text_joint_to_text --task speech_text_joint_denoising \ - --criterion speech_text_pretrain_cross_entropy --optimizer adam --weight-decay 0.01 --config-yaml config_s2p.yaml --config-s2s-yaml config.yaml --ddp-backend no_c10d \ - --lang-pairs pho-wrd --num-workers 4 --log-interval 500 --save-interval-updates 5000 --keep-interval-updates 1 --no-emb-update-unsup --report-accuracy --lr 0.001 --end-learning-rate 1e-06 \ - --lr-scheduler polynomial_decay --warmup-updates 10000 --total-num-update 800000 --update-freq 6 --validate-interval-updates 10000 --train-subset train \ - --valid-subset valid,valid_sup_speech,valid_sup_speech_s2s,valid_unsup_speech --dataset-impl mmap \ - --sup-speech-data $S2P_DATA_PATH --sup-speech-train-subset train_960.ali --sup-speech-valid-subset dev-clean.ali --sup-speech-s2s-data $S2T_DATA_PATH \ - --sup-speech-s2s-train-subset train --sup-speech-s2s-valid-subset dev-clean --unsup-speech-train-data $SSL_DATA_PATH/train.tsv --unsup-speech-valid-data $SSL_DATA_PATH/valid.tsv \ - --batch-size 200 --batch-size-valid 150 --max-source-positions 1024 --max-target-positions 1024 --max-text-tokens 3072 --max-speech-positions 600000 \ - --max-sample-size 750000 --min-sample-size 64000 --max-speech-tokens 750000 --max-tokens-valid 750000 --skip-invalid-size-inputs-valid-test \ - --unsupervised-speech-sample-ratio 3.0 --supervised-speech-sample-ratio 5 --supervised-speech-s2s-sample-ratio 5 --text-sample-ratio 1.0 --mask 0.3 --mask-random 0.1 \ - --mask-length span-poisson --speech-sup-mask-prob 0.3 --speech-unsup-mask-prob 0.7 --use-mask-whole-words --arch speech_text_pretrain_bart_base_stack \ - --no-scale-feature --activation-fn gelu --speech-extractor-mode default --stacked-encoder all --encoder-normalize-before --decoder-normalize-before \ - --encoder-learned-pos --decoder-learned-pos --dropout 0.1 --load-pretrained-mbart-encoder-from $BART --load-pretrained-mbart-decoder-from $BART -``` -The current implementation also supports model pre-training without the forced alignment supervised data. In this case, CTC is used to optimize the S2P task. We need to do following changes for the setting: -1. options to be added -``` ---use-sup-speech-ctc --criterion speech_text_pretrain_compound -``` -2. options to be deleted -``` ---same-data-update --criterion speech_text_pretrain_cross_entropy -``` -However, we find the CTC based pre-training is still worse than the forced alignment based setting. It could be partially due to the inferior pre-training setting that we re-use the forced alignment based pre-training setting for the CTC based pre-training. - -#### Fine-tuning -``` -python train.py $S2T_DATA_PATH \ - --save-dir $SAVE_FT_PATH --num-workers 8 --task speech_text_joint_to_text --arch dualinputs2twavtransformer_base_stack \ - --user-dir examples/speech_text_joint_to_text --max-update 100000 --optimizer adam --lr-scheduler inverse_sqrt --lr 0.0003 --update-freq 3 --clip-norm 10.0 \ - --criterion guided_label_smoothed_cross_entropy_with_accuracy --guide-alpha 0.8 --label-smoothing 0.1 --warmup-updates 20000 --attentive-cost-regularization 0.02 \ - --enc-grad-mult 2.0 --max-tokens 800000 --max-source-positions 800000 --max-tokens-text 10000 --max-positions-text 1024 --max-target-positions 1024 --no-scale-feature \ - --activation-fn gelu --load-pretrained-speech-text-encoder $SAVE_PRE_PATH/checkpoint_last.pt --load-pretrained-speech-text-decoder $SAVE_PRE_PATH/checkpoint_last.pt \ - --encoder-normalize-before --decoder-normalize-before --speech-extractor-mode default --speech-mask-channel-length 64 --speech-mask-channel-prob 0.5 \ - --speech-mask-length 10 --speech-mask-prob 0.65 --text-sample-ratio 0.25 --mask-text-ratio 0.3 --mask-text-type random --parallel-text-data text_bin \ - --text-input-cost-ratio 0.5 --langpairs pho-wrd --update-mix-data --log-format json --max-tokens-valid 800000 --ddp-backend no_c10d --log-interval 500 \ - --config-yaml config.yaml --skip-invalid-size-inputs-valid-test --keep-last-epochs 50 --layernorm-embedding --encoder-learned-pos --decoder-learned-pos -``` - -### Evaluation -The last 10 epoch models from fine-tuning is conducted model average to get $FINAL_MODEL -``` -python ./fairseq_cli/generate.py \ - $S2T_DATA_PATH \ - --task speech_text_joint_to_text \ - --max-tokens 800000 \ - --max-source-positions 800000 \ - --nbest 1 \ - --results-path $RESULTS_LOG \ - --batch-size 512 \ - --path $FINAL_MODEL \ - --gen-subset $SUBSET \ - --config-yaml config.yaml \ - --scoring wer \ - --beam 10 --lenpen 1.0 examples/speech_text_joint_to_text \ - --user-dir examples/speech_text_joint_to_text --load-speech-only \ - --model-overrides {'load_pretrained_speech_text_decoder':'','load_pretrained_speech_text_encoder':''} -``` - -### Results and models -| | dev-clean | dev-other | test-clean | test-other | -|---|---|---|---|---| -| WER| 2.0 | 4.4 | 2.1 |4.6 | - -**Model Links**: -- [config_s2p.yaml](https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/acl2022/librispeech/pretrain/config_s2p.yaml): Config for S2P -- [spm.model](https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/acl2022/librispeech/finetuned/spm.model): Sentence Piece model -- [src_dict.txt](https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/acl2022/librispeech/finetuned/src_dict.txt): Source Phoneme Dictionary -- [tgt_dict.txt](https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/acl2022/librispeech/finetuned/tgt_dict.txt): Target Sentence Piece Dictionary -- [config.yaml](https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/acl2022/librispeech/finetuned/config.yaml): Config for S2T -- [BART](https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/acl2022/librispeech/pretrain/bart.pt): trained from Librispeech text data -- [Joint Pre-trained model](https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/acl2022/librispeech/pretrain/checkpoint6.pt): model pre-trained with 960 hours Librispeech data (S2P, S2T) Librispeech text training data (T2T) and Librilight data (SSL) -- [Fine-tuned model](https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/acl2022/librispeech/finetuned/checkpoint_ave_10.pt): the pre-trained model is fined one 960 hours Librispeech speech and text data. (S2T + T2T) - -## MuST-C -### Prepare Data -Compared with the ASR Librispeech ASR recipe, the differences are below: -- Replace the speech data with corresponding MuST-C data -- Parallel text data from WMT is replaced the Librispeech text data - -### Model Build -#### Pre-training -EN-DE is used as an example -``` -python train.py $TXT_DATA \ - --save-dir $SAVE_PRE_PATH --user-dir examples/speech_text_joint_to_text --task speech_text_joint_denoising --criterion speech_text_pretrain_cross_entropy --optimizer adam --weight-decay 0.01 \ - --config-yaml config_s2p.yaml --config-s2s-yaml config.yaml --ddp-backend no_c10d --lang-pairs-bitext en-fr --num-workers 4 --log-interval 500 --save-interval-updates 5000 --keep-interval-updates 1 \ - --no-emb-update-unsup --use-decoder-output-proj --report-accuracy --lr 0.001 --end-learning-rate 1e-06 --lr-scheduler polynomial_decay --warmup-updates 10000 --total-num-update 800000 \ - --update-freq 8 --validate-interval-updates 10000 --train-subset train --valid-subset valid_sup_speech,valid_sup_speech_s2s,valid_unsup_speech --dataset-impl mmap \ - --sup-speech-data $S2P_DATA_PATH --sup-speech-train-subset train --sup-speech-valid-subset dev --sup-speech-s2s-data $S2T_DATA_PATH --sup-speech-s2s-train-subset train \ - --sup-speech-s2s-valid-subset dev --unsup-speech-train-data $SSL_DATA_PATH/train.tsv --unsup-speech-valid-data $SSL_DATA_PATH/valid.tsv --batch-size 200 --batch-size-valid 100 \ - --max-source-positions 1024 --max-target-positions 1024 --max-text-tokens 2048 --max-speech-positions 600000 --max-sample-size 600000 --min-sample-size 64000 \ - --max-speech-tokens 600000 --max-tokens-valid 600000 --skip-invalid-size-inputs-valid-test --unsupervised-speech-sample-ratio 1.2 --supervised-speech-sample-ratio 10 \ - --supervised-speech-s2s-sample-ratio 10 --bitext-sample-ratio 0.5 --mask 0.3 --mask-random 0.1 --mask-length span-poisson --speech-sup-mask-prob 0.3 \ - --speech-unsup-mask-prob 0.7 --use-mask-whole-words --arch speech_text_pretrain_bart_base_stack --no-scale-feature --activation-fn gelu --speech-extractor-mode default \ - --stacked-encoder s2s --encoder-normalize-before --decoder-normalize-before --encoder-learned-pos --decoder-learned-pos --dropout 0.1 \ - --load-pretrained-mbart-encoder-from $EN_FR_NMT --load-pretrained-mbart-decoder-from $EN_FR_NMT -``` -#### Fine-tuning -``` -python train.py $S2T_DATA_PATH \ - --save-dir $SAVE_FT_PATH --num-workers 8 --task speech_text_joint_to_text --arch dualinputs2twavtransformer_base_stack --user-dir examples/speech_text_joint_to_text \ - --max-epoch 25 --update-mix-data --optimizer adam --lr-scheduler inverse_sqrt --lr 0.0003 --update-freq 4 --clip-norm 10.0 --warmup-updates 20000 \ - --criterion guided_label_smoothed_cross_entropy_with_accuracy --guide-alpha 0.8 --attentive-cost-regularization 0.02 --enc-grad-mult 2.0 --label-smoothing 0.1 \ - --max-tokens 800000 --max-source-positions 800000 --max-tokens-text 10000 --max-positions-text 1024 --load-pretrained-speech-text-encoder $SAVE_PRE_PATH/checkpoint_last.pt \ - --load-pretrained-speech-text-decoder $SAVE_PRE_PATH/checkpoint_last.pt --speech-mask-channel-length 64 --speech-mask-channel-prob 0.5 --speech-mask-length 10 \ - --speech-mask-prob 0.65 --text-sample-ratio 0.05 --mask-text-ratio 0.3 --mask-text-type random --parallel-text-data data-bin-wt --text-input-cost-ratio 0.5 \ - --langpairs en-fr --log-format json --max-tokens-valid 800000 --ddp-backend no_c10d --log-interval 100 --config-yaml config.yaml --skip-invalid-size-inputs-valid-test \ - --noise-token '▁NOISE' --keep-last-epochs 40 --layernorm-embedding --encoder-learned-pos --decoder-learned-pos --activation-fn gelu \ - --speech-extractor-mode default --max-target-positions 1024 --encoder-normalize-before --decoder-normalize-before -``` - -### Evaluation -The last 10 epoch models from fine-tuning is conducted model average to get $FINAL_MODEL -``` -python fairseq_cli/generate.py \ - $S2T_DATA_PATH \ - --task speech_text_joint_to_text \ - --nbest 1 \ - --max-tokens 800000 \ - --max-source-positions 800000 \ - --results-path $RESULTS_LOG \ - --batch-size 512 \ - --path $FINAL_MODEL \ - --gen-subset $SUBSET \ - --config-yaml config.yaml \ - --scoring sacrebleu \ - --beam 10 --lenpen 1.0 examples/speech_text_joint_to_text \ - --user-dir examples/speech_text_joint_to_text --load-speech-only \ - --model-overrides {'load_pretrained_speech_text_decoder':'','load_pretrained_speech_text_encoder':''} -``` - - -### Results and models -| | en-fr | en-es | en-de | -|---|---|---|---| -| BLEU| 39.7 | 33.2 |29.2 | - - -**Model Links**: -1. DE - - [de config.yaml](https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/acl2022/must_c/de/config.yaml) - - [de src_dict.txt](https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/acl2022/must_c/de/src_dict.txt) - - [de tgt_dict.txt](https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/acl2022/must_c/de/tgt_dict.txt) - - [de spm.model](https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/acl2022/must_c/de/spm.model) - - [de pre-trained nmt model](https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/acl2022/must_c/de/nmt.pt) - - [de pre-trained model](https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/acl2022/must_c/de/checkpoint_pretraing.pt) - - [de fine-tuned model](https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/acl2022/must_c/de/checkpoint_finetune_ave10.pt) -2. ES - - [es config.yaml](https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/acl2022/must_c/es/config.yaml) - - [es src_dict.txt](https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/acl2022/must_c/es/src_dict.txt) - - [es tgt_dict.txt](https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/acl2022/must_c/es/tgt_dict.txt) - - [es spm.model](https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/acl2022/must_c/es/spm.model) - - [es pre-trained nmt model](https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/acl2022/must_c/es/nmt.pt) - - [es pre-trained model](https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/acl2022/must_c/es/checkpoint_pretraing.pt) - - [es fine-tuned model](https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/acl2022/must_c/es/checkpoint_finetune_ave10.pt) -3. FR - - [fr config.yaml](https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/acl2022/must_c/fr/config.yaml) - - [fr src_dict.txt](https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/acl2022/must_c/fr/src_dict.txt) - - [fr tgt_dict.txt](https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/acl2022/must_c/fr/tgt_dict.txt) - - [fr spm.model](https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/acl2022/must_c/fr/spm.model) - - [fr pre-trained nmt model](https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/acl2022/must_c/fr/nmt.pt) - - [fr pre-trained model](https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/acl2022/must_c/fr/checkpoint_pretraing.pt) - - [fr fine-tuned model](https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/acl2022/must_c/fr/checkpoint_finetune_ave10.pt) -4. [config_s2p.yaml](https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/acl2022/must_c/config_s2p.yaml) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/models/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/models/__init__.py deleted file mode 100644 index 5fc5d9e21b2b5d272bd67610958efbdb1eecb158..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/models/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import importlib -import os - diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/models/joint_speech_text_pretrain_transformer.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/models/joint_speech_text_pretrain_transformer.py deleted file mode 100644 index 0490bd97cf64884e486e53edb4dda28c85f1df94..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/models/joint_speech_text_pretrain_transformer.py +++ /dev/null @@ -1,697 +0,0 @@ -#!/usr/bin/env python3 - -import logging -from collections import OrderedDict, namedtuple -from typing import Dict, Optional - -import torch -import torch.nn as nn -import torch.nn.functional as F -from fairseq import checkpoint_utils, utils -from fairseq.file_io import PathManager -from fairseq.models import ( - FairseqDecoder, - FairseqEncoderDecoderModel, - register_model, - register_model_architecture, -) -from fairseq.models.speech_to_text import ( - MultiInputDecoder, - MultiModalityEncoder, - SpeechWavTransformerEncoder, - StackedSpeechWavTransformerEncoder, -) -from fairseq.models.transformer import ( - TransformerDecoder, - TransformerEncoder, - TransformerModel, -) -from torch import Tensor - -logger = logging.getLogger(__name__) - - -class SpeechTextPreTrainEncoder(MultiModalityEncoder): - def __init__( - self, - dictionary, - sup_speech_encoder, - sup_s2s_speech_encoder, - unsup_speech_encoder, - text_encoder, - ): - super().__init__(dictionary) - self.sup_speech_encoder = sup_speech_encoder - self.sup_s2s_speech_encoder = sup_s2s_speech_encoder - self.unsup_speech_encoder = unsup_speech_encoder - self.text_encoder = text_encoder - - @classmethod - def update_transformer_encoder_cfg(cls, args, update_dict): - cfg = dict(args._get_kwargs()) - for fkey in update_dict.keys(): - cfg[fkey] = update_dict[fkey] - cfg.pop("_name", None) # remove keys start with _ - model_args = namedtuple("args", cfg.keys())(*cfg.values()) - return model_args - - @classmethod - def build_text_encoder(cls, args, src_dictionary): - enc_emb = nn.Embedding( - len(src_dictionary), args.encoder_embed_dim, src_dictionary.pad() - ) - model_args = cls.update_transformer_encoder_cfg( - args, {"encoder_layers": args.text_encoder_layers} - ) - text_encoder = TransformerEncoder(model_args, src_dictionary, enc_emb) - return text_encoder - - @classmethod - def build_speech_encoder(cls, args): - model_args = cls.update_transformer_encoder_cfg( - args, - { - "encoder_layers": args.speech_encoder_layers, - "speech_mask_prob": args.speech_sup_mask_prob, - }, - ) - speech_encoder = SpeechWavTransformerEncoder(model_args) - return speech_encoder - - @classmethod - def share_layers(cls, src_layers, tgt_layers): # share layer but not dropout - # share parameters in src_layers with tgt_layers - assert len(src_layers) == len(tgt_layers) - for i, ly in enumerate(src_layers): - tly = tgt_layers[i] - tly.self_attn = ly.self_attn - tly.self_attn_layer_norm = ly.self_attn_layer_norm - tly.activation_fn = ly.activation_fn - tly.normalize_before = ly.normalize_before - tly.fc1 = ly.fc1 - tly.fc2 = ly.fc2 - tly.final_layer_norm = ly.final_layer_norm - if hasattr(tly, "encoder_attn"): - tly.encoder_attn = ly.encoder_attn - tly.encoder_attn_layer_norm = ly.encoder_attn_layer_norm - return tgt_layers - - @classmethod - def build_unsup_speech_encoder(cls, args, sup_speech_encoder): - model_args = cls.update_transformer_encoder_cfg( - args, - { - "encoder_layers": args.speech_encoder_layers, - "speech_mask_prob": args.speech_unsup_mask_prob, - "encoder_layerdrop": 0.0, - "decoder_layerdrop": 0.0, - "dropout": args.speech_unsup_dropout, - "activation_dropout": args.speech_unsup_dropout, - "attention_dropout": 0.0, - "dropout_features": args.speech_unsup_feature_dropout, - "dropout_input": args.speech_unsup_feature_dropout, - }, - ) - - unsup_speech_encoder = SpeechWavTransformerEncoder(model_args, alway_mask=True) - unsup_speech_encoder.layer_norm = sup_speech_encoder.layer_norm - unsup_speech_encoder.layers = cls.share_layers( - sup_speech_encoder.layers, unsup_speech_encoder.layers - ) - unsup_speech_encoder.mask_emb = sup_speech_encoder.mask_emb - unsup_speech_encoder.embed_positions = sup_speech_encoder.embed_positions - unsup_speech_encoder.feat_layer_norm = sup_speech_encoder.feat_layer_norm - unsup_speech_encoder.feat_proj = sup_speech_encoder.feat_proj - unsup_speech_encoder.subsample = sup_speech_encoder.subsample - return unsup_speech_encoder - - @classmethod - def build_encoder(cls, args, dictionary): - text_encoder = cls.build_text_encoder(args, dictionary) - if getattr(args, "load_pretrained_mbart_encoder_from", None): - text_encoder = checkpoint_utils.load_pretrained_component_from_model( - component=text_encoder, - checkpoint=args.load_pretrained_mbart_encoder_from, - ) - speech_encoder = cls.build_speech_encoder(args) - if getattr(args, "load_pretrained_feature_extractor_from", None): - - def load_feature_extractor(component, checkpoint): - if not PathManager.exists(checkpoint): - raise IOError("Model file not found: {}".format(checkpoint)) - state = checkpoint_utils.load_checkpoint_to_cpu(checkpoint) - component_state_dict = OrderedDict() - - component_prefix = "feature_extractor" - for key in state["model"].keys(): - if key.startswith(component_prefix): - component_subkey = key[len(component_prefix) + 1 :] - component_state_dict[component_subkey] = state["model"][key] - component.load_state_dict(component_state_dict, strict=True) - return component - - speech_encoder.subsample = load_feature_extractor( - speech_encoder.subsample, args.load_pretrained_feature_extractor_from - ) - speech_s2s_encoder = speech_encoder - unsup_speech_encoder = cls.build_unsup_speech_encoder(args, speech_encoder) - if getattr(args, "stacked_encoder", "none") != "none": - if args.encoder_shared_text_layers_from_begin > 0: - raise ValueError( - "We can not stack encoders and share encoders at the same time!" - ) - speech_s2s_encoder = StackedSpeechWavTransformerEncoder( - speech_encoder, text_encoder.layers, text_encoder.layer_norm - ) - if args.stacked_encoder == "all": - speech_encoder = speech_s2s_encoder - unsup_speech_encoder = StackedSpeechWavTransformerEncoder( - unsup_speech_encoder, text_encoder.layers, text_encoder.layer_norm - ) - else: - cls.share_speech_text_encoder( - speech_encoder, text_encoder, args.encoder_shared_text_layers_from_begin - ) - return SpeechTextPreTrainEncoder( - dictionary, - speech_encoder, - speech_s2s_encoder, - unsup_speech_encoder, - text_encoder, - ) - - @classmethod - def share_speech_text_encoder( - cls, speech_encoder, text_encoder, shared_layers_from_begin - ): - if shared_layers_from_begin > 0: - num_text_encoder_layers = len(text_encoder.layers) - assert len(speech_encoder.layers) >= shared_layers_from_begin - assert num_text_encoder_layers >= shared_layers_from_begin - assert len(speech_encoder.layers) >= num_text_encoder_layers - for i, ly in enumerate( - speech_encoder.layers[ - -num_text_encoder_layers : -num_text_encoder_layers - + shared_layers_from_begin - ] - ): - assert isinstance(text_encoder.layers[i], type(ly)) - text_encoder.layers[i] = ly - - def select_encoder(self, mode, **kwargs): - if mode in ("speech", "sup_speech_ctc", "sup_speech_ali", "sup_speech_s2s"): - kwargs["features_only"] = True - if mode == "sup_speech_s2s": - return self.sup_s2s_speech_encoder, kwargs - return self.sup_speech_encoder, kwargs - elif mode == "unsup_speech": - kwargs["features_only"] = False - return self.unsup_speech_encoder, kwargs - elif mode in ("text", "bitext"): - return self.text_encoder, kwargs - else: - raise NotImplementedError(f"{mode} is not supported") - return None, kwargs - - def forward(self, src_tokens, src_lengths=None, mode="", alignment=None, **kwargs): - return super().forward(src_tokens, src_lengths, mode, **kwargs) - - -# SpeechDummyDecoder works as an extension of encoder, so we could fit encoder only training into seq2seq training -class SpeechDummyDecoder(FairseqDecoder): - def __init__( - self, - dictionary, - output_embedding, - no_emb_update_unsup=False, - use_output_proj=False, - ): - super().__init__(dictionary) - self.output_embedding = output_embedding - num_embedding, num_dim = self.output_embedding.weight.size() - self.out_proj = ( - None if use_output_proj is False else nn.Linear(num_dim, num_dim) - ) - self.no_emb_update_unsup = no_emb_update_unsup - - def extend_alignment(self, alignment, src_lengths, prev_output_tokens): - # alignment: B X N - # src_lengths: B X T - # prev_output_tokens: B X (N + 1) - tgt_tokens = prev_output_tokens[ - :, 1: - ] # remove the leading start of sentence token - ext_alignment = ( - torch.ones(len(src_lengths), src_lengths.max(), device=src_lengths.device) - .long() - .fill_(self.dictionary.pad()) - ) - for bs in range(src_lengths.size(0)): - tgt_length = tgt_tokens[bs].ne(self.dictionary.pad()).sum().item() - assert tgt_length == sum(alignment[bs].ne(1)) + 1 - src_st = 0 - for i in range(tgt_length): - tok = tgt_tokens[bs][i] - src_ed = (alignment[bs][i] * src_lengths[bs]).int().item() - ext_alignment[bs][src_st:src_ed].fill_(tok) - src_st = src_ed - return ext_alignment - - def forward( - self, - prev_output_tokens, - encoder_out, - incremental_state=None, - mode="speech", - alignment=None, - **kwargs, - ): - """ - Args: - prev_output_tokens (LongTensor): previous decoder outputs of shape - `(batch, tgt_len)`, for teacher forcing - encoder_out (optional): output from the encoder, used for - encoder-side attention - incremental_state (dict): dictionary used for storing state during - :ref:`Incremental decoding` - features_only (bool, optional): only return features without - applying output layer (default: False). - full_context_alignment (bool, optional): don't apply - auto-regressive mask to self-attention (default: False). - - Returns: - sup_speech_ctc: - dictionary{"logits": logits, "padding_mask": padding_mask} - sup_speech_ali and unsup_speech: - tuple: - - the decoder's output of shape `(batch, tgt_len, vocab)` - - a dictionary with any model-specific outputs - """ - emb_weight = self.output_embedding.weight - if ( - mode == "unsup_speech" and self.no_emb_update_unsup - ): # no gradient for embedding here - emb_weight = emb_weight.detach() - enc_out = ( - encoder_out["encoder_out"][0] - if self.out_proj is None - else self.out_proj(encoder_out["encoder_out"][0]) - ) - logits = F.linear(enc_out, emb_weight, None).transpose(0, 1) # B X T X C - others = None - if mode in ( - "speech", - "sup_speech_ctc", - ): # speech data with label, do forcealignment - if len(encoder_out["encoder_padding_mask"]) > 0: - padding_mask = encoder_out["encoder_padding_mask"][0] - logits = logits.masked_fill(padding_mask, float("-inf")) - else: - seq_len, bsz = encoder_out["encoder_out"][0].size()[:2] - padding_mask = torch.zeros( - bsz, seq_len, device=encoder_out["encoder_out"][0].device - ).bool() - return {"x": logits, "padding_mask": padding_mask} - elif mode == "sup_speech_ali": - src_lengths = None - if len(encoder_out["encoder_padding_mask"]) > 0: - src_lengths = (1 - encoder_out["encoder_padding_mask"][0].long()).sum( - -1 - ) - else: - seq_len, bsz = encoder_out["encoder_out"][0].size()[:2] - src_lengths = ( - torch.ones(bsz, device=encoder_out["encoder_out"][0].device).long() - * seq_len - ) - assert alignment is not None - alignment = self.extend_alignment( - alignment, src_lengths, prev_output_tokens - ) - others = {"pseudo_target_tokens": alignment} - elif mode == "unsup_speech": - enc_out_ori = ( - encoder_out["encoder_unmasked_out"][0] - if self.out_proj is None - else self.out_proj(encoder_out["encoder_unmasked_out"][0]) - ) - logits_ori = F.linear(enc_out_ori, emb_weight, None).transpose(0, 1) - if len(encoder_out["encoder_padding_mask"]) > 0: - encoder_padding_mask = encoder_out["encoder_padding_mask"][0] - logits_ori = logits_ori.masked_fill(encoder_padding_mask, float("-inf")) - pseudo_labels = utils.log_softmax(logits_ori, dim=-1) - others = { - "pseudo_target_logprobs": pseudo_labels, - "padding_mask": encoder_out["encoder_padding_mask"], # B X T - "mask_indices": encoder_out[ - "mask_indices" - ], # True for masked frames B X T - } - return logits, others - - def get_normalized_probs( - self, - net_output: Dict[str, Tensor], - log_probs: bool, - sample: Optional[Dict[str, Tensor]] = None, - ): - return self.get_normalized_probs_scriptable( - (net_output["x"], None), log_probs, sample - ) - - -class SpeechTextPreTrainDecoder(MultiInputDecoder): - def __init__(self, dictionary, speech_decoder, text_decoder): - super().__init__(dictionary) - self.speech_decoder = speech_decoder - self.text_decoder = text_decoder - - def select_decoder(self, mode, **kwargs): - if mode == "unsup_speech": - kwargs["mode"] = mode - return self.speech_decoder, kwargs - if mode in ("text", "bitext"): - return self.text_decoder, kwargs - if mode in ("speech", "sup_speech_ctc", "sup_speech_ali"): - kwargs["mode"] = mode - return self.speech_decoder, kwargs - if mode in ("speech", "sup_speech_s2s"): - if "alignment" in kwargs: - del kwargs["alignment"] - return self.text_decoder, kwargs - - raise NotImplementedError(f"{mode} is not supported") - return None, kwargs - - def get_normalized_probs( - self, - net_output, - log_probs, - sample=None, - ): - """Get normalized probabilities (or log probs) from a net's output.""" - if isinstance(net_output, dict): - return self.speech_decoder.get_normalized_probs( - net_output, log_probs, sample - ) - return self.text_decoder.get_normalized_probs(net_output, log_probs, sample) - - @classmethod - def build_text_decoder(cls, args, tgt_dictionary, dec_emb_share=None): - dec_emb = ( - nn.Embedding( - len(tgt_dictionary), args.decoder_embed_dim, tgt_dictionary.pad() - ) - if dec_emb_share is None - else dec_emb_share - ) - text_decoder = TransformerDecoder(args, tgt_dictionary, dec_emb) - return text_decoder - - @classmethod - def build_dummy_speech_decoder(cls, args, dictionary, dec_emb_share=None): - dec_emb = ( - nn.Embedding(len(dictionary), args.decoder_embed_dim, dictionary.pad()) - if dec_emb_share is None - else dec_emb_share - ) - speech_decoder = SpeechDummyDecoder( - dictionary, - dec_emb, - no_emb_update_unsup=getattr(args, "no_emb_update_unsup", False), - use_output_proj=getattr(args, "use_decoder_output_proj", False), - ) - return speech_decoder - - @classmethod - def build_decoder( - cls, args, text_dictionary, speech_dictionary, speech_output_embedding - ): - text_decoder = cls.build_text_decoder(args, text_dictionary) - speech_decoder = cls.build_dummy_speech_decoder( - args, speech_dictionary, speech_output_embedding - ) - if getattr(args, "load_pretrained_mbart_decoder_from", None): - text_decoder = checkpoint_utils.load_pretrained_component_from_model( - component=text_decoder, - checkpoint=args.load_pretrained_mbart_decoder_from, - ) - return SpeechTextPreTrainDecoder(text_dictionary, speech_decoder, text_decoder) - - -@register_model("speech_text_pretrain_bart") -class SpeechTextPreTrainModel(FairseqEncoderDecoderModel): - def __init__(self, encoder, decoder): - super().__init__(encoder, decoder) - self.num_updates = 0 - - def forward( - self, src_tokens, src_lengths, prev_output_tokens, src_lang_ids=None, **kwargs - ): - if src_lang_ids is not None: - encoder_out = self.encoder( - src_tokens, src_lengths=src_lengths, src_lang_ids=src_lang_ids, **kwargs - ) - else: - encoder_out = self.encoder(src_tokens, src_lengths=src_lengths, **kwargs) - decoder_out = self.decoder( - prev_output_tokens, encoder_out=encoder_out, **kwargs - ) - return decoder_out - - def max_positions(self): - return None # it is provided in task - - def get_targets(self, sample, net_output): - mode = sample["net_input"]["mode"] - if mode == "unsup_speech": - return {"target_logprobs": net_output[1]["pseudo_target_logprobs"]} - if mode == "sup_speech_ali": - return net_output[1]["pseudo_target_tokens"] - return sample["target"] - - def get_normalized_probs( - self, - net_output, - log_probs, - sample=None, - ): - # net_output['encoder_out'] is a (B, T, D) tensor - lprobs = self.get_normalized_probs_scriptable(net_output, log_probs, sample) - lprobs.batch_first = True - return lprobs - - @staticmethod - def add_args(parser): - TransformerModel.add_args(parser) - SpeechWavTransformerEncoder.add_args(parser) - parser.add_argument( - "--speech-sup-mask-prob", - type=float, - help="probability of replacing a token with mask (sup-speech)", - ) - parser.add_argument( - "--speech-unsup-mask-prob", - type=float, - help="probability of replacing a token with mask (unsup-speech)", - ) - parser.add_argument( - "--load-pretrained-mbart-encoder-from", - type=str, - metavar="STR", - help="model to take text encoder weights from (for initialization)", - ) - - parser.add_argument( - "--load-pretrained-mbart-decoder-from", - type=str, - metavar="STR", - help="model to take text decoder weights from (for initialization)", - ) - - parser.add_argument( - "--load-pretrained-feature-extractor-from", - type=str, - metavar="STR", - help="model to take feature extractor weights from (for initialization)", - ) - - parser.add_argument( - "--speech-unsup-dropout", - type=float, - default=0, - help="dropout for unsupervised speech encoder", - ) - - parser.add_argument( - "--speech-unsup-feature-dropout", - type=float, - default=0, - help="dropout for unsupervised speech feature encoder", - ) - - parser.add_argument( - "--encoder-shared-text-layers-from-begin", - type=int, - help="number of text encoder layers shared with speech encoder (from first layer)", - ) - - parser.add_argument( - "--stacked-encoder", - default="none", - choices=["none", "s2s", "all"], - help="stack speech and text encoders", - ) - - parser.add_argument("--use-decoder-output-proj", action="store_true") - - @classmethod - def build_model(cls, args, task): - encoder = SpeechTextPreTrainEncoder.build_encoder(args, task.src_dict) - decoder = SpeechTextPreTrainDecoder.build_decoder( - args, task.tgt_dict, task.src_dict, encoder.text_encoder.embed_tokens - ) - model = SpeechTextPreTrainModel(encoder, decoder) - return model - - def upgrade_state_dict(self, state_dict): - """Upgrade old state dicts to work with newer code.""" - if "decoder.speech_decoder.output_projection.weight" in state_dict: - del state_dict["decoder.speech_decoder.output_projection.weight"] - self.upgrade_state_dict_named(state_dict, "") - - -@register_model_architecture( - "speech_text_pretrain_bart", "speech_text_pretrain_bart_base" -) -def speech_text_pretrain_bart_base(args): - # speech masking - args.dropout_input = getattr(args, "dropout_input", 0) - args.dropout_features = getattr(args, "dropout_features", 0) - args.speech_mask_length = getattr(args, "speech_mask_length", 10) - args.speech_mask_prob = getattr(args, "speech_mask_prob", 0.65) - args.speech_sup_mask_prob = getattr(args, "speech_sup_mask_prob", 0.3) - args.speech_unsup_mask_prob = getattr( - args, "speech_unsup_mask_prob", args.speech_mask_prob - ) - args.speech_mask_selection = getattr(args, "speech_mask_selection", "static") - args.speech_mask_other = getattr(args, "speech_mask_other", 0) - args.speech_mask_min_space = getattr(args, "speech_mask_min_space", 1) - args.speech_no_mask_overlap = getattr(args, "speech_no_mask_overlap", False) - - args.speech_mask_channel_length = getattr(args, "speech_mask_channel_length", 10) - args.speech_mask_channel_prob = getattr(args, "speech_mask_channel_prob", 0.0) - args.speech_mask_channel_selection = getattr( - args, "speech_mask_channel_selection", "static" - ) - args.speech_mask_channel_other = getattr(args, "speech_mask_channel_other", 0) - args.speech_mask_channel_min_space = getattr( - args, "speech_mask_channel_min_space", 1 - ) - args.speech_no_mask_channel_overlap = getattr( - args, "speech_no_mask_channel_overlap", False - ) - args.no_scale_feature = getattr(args, "", False) - args.feature_grad_mult = getattr(args, "feature_grad_mult", 1.0) # 0.1 - - # Transformer - args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 768) - args.encoder_ffn_embed_dim = getattr( - args, "encoder_ffn_embed_dim", args.encoder_embed_dim * 4 - ) - args.encoder_attention_heads = getattr(args, "encoder_attention_heads", 12) - args.encoder_normalize_before = getattr(args, "encoder_normalize_before", False) - args.encoder_layerdrop = getattr(args, "encoder_layerdrop", 0) - args.encoder_learned_pos = getattr(args, "encoder_learned_pos", False) - args.speech_conv_bias = getattr(args, "speech_conv_bias", False) - - args.decoder_embed_dim = getattr(args, "decoder_embed_dim", args.encoder_embed_dim) - args.decoder_ffn_embed_dim = getattr( - args, "decoder_ffn_embed_dim", args.encoder_ffn_embed_dim - ) - args.decoder_attention_heads = getattr( - args, "decoder_attention_heads", args.encoder_attention_heads - ) - args.decoder_normalize_before = getattr(args, "decoder_normalize_before", False) - args.decoder_learned_pos = getattr(args, "decoder_learned_pos", False) - args.dropout = getattr(args, "dropout", 0.1) - args.attention_dropout = getattr(args, "attention_dropout", args.dropout) - args.activation_dropout = getattr(args, "activation_dropout", 0.0) - args.activation_fn = getattr(args, "activation_fn", "relu") # gelu? - args.adaptive_softmax_cutoff = getattr(args, "adaptive_softmax_cutoff", None) - args.adaptive_softmax_dropout = getattr(args, "adaptive_softmax_dropout", 0) - - args.speech_unsup_dropout = getattr(args, "speech_unsup_dropout", 0) - args.speech_unsup_feature_dropout = getattr(args, "speech_unsup_feature_dropout", 0) - - args.tie_adaptive_weights = getattr(args, "tie_adaptive_weights", False) - args.share_decoder_input_output_embed = getattr( - args, "share_decoder_input_output_embed", False - ) - args.no_token_positional_embeddings = getattr( - args, "no_token_positional_embeddings", False - ) - args.adaptive_input = getattr(args, "adaptive_input", False) - args.decoder_layerdrop = getattr(args, "decoder_layerdrop", 0.0) - args.decoder_output_dim = getattr( - args, "decoder_output_dim", args.decoder_embed_dim - ) - args.layernorm_embedding = getattr(args, "layernorm_embedding", False) - args.no_scale_embedding = getattr(args, "no_scale_embedding", False) - args.quant_noise_pq = getattr(args, "quant_noise_pq", 0) - - args.speech_encoder_layers = getattr(args, "speech_encoder_layers", 12) - args.text_encoder_layers = getattr(args, "text_encoder_layers", 6) - args.encoder_shared_text_layers_from_begin = getattr( - args, "encoder_shared_text_layers_from_begin", 6 - ) - args.decoder_layers = getattr(args, "decoder_layers", 6) - - args.no_emb_update_unsup = getattr(args, "no_emb_update_unsup", False) - - -@register_model_architecture( - "speech_text_pretrain_bart", "speech_text_pretrain_bart_base_stack" -) -def speech_text_pretrain_bart_base_stack(args): - args.speech_encoder_layers = getattr(args, "speech_encoder_layers", 6) - args.text_encoder_layers = getattr(args, "text_encoder_layers", 6) - args.encoder_shared_text_layers_from_begin = getattr( - args, "encoder_shared_text_layers_from_begin", 0 - ) - args.stacked_encoder = getattr(args, "stacked_encoder", "all") - args.layernorm_embedding = getattr(args, "layernorm_embedding", True) - speech_text_pretrain_bart_base(args) - - -@register_model_architecture( - "speech_text_pretrain_bart", "speech_text_pretrain_bart_large" -) -def speech_text_pretrain_bart_large(args): - args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 1024) - args.encoder_attention_heads = getattr(args, "encoder_attention_heads", 16) - args.speech_encoder_layers = getattr(args, "speech_encoder_layers", 24) - args.text_encoder_layers = getattr(args, "text_encoder_layers", 12) - args.encoder_shared_text_layers_from_begin = getattr( - args, "encoder_shared_text_layers_from_begin", 12 - ) - args.decoder_layers = getattr(args, "decoder_layers", 12) - args.dropout = getattr(args, "dropout", 0.3) - speech_text_pretrain_bart_base(args) - - -@register_model_architecture( - "speech_text_pretrain_bart", "speech_text_pretrain_bart_large_stack" -) -def speech_text_pretrain_bart_large_stack(args): - args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 1024) - args.encoder_attention_heads = getattr(args, "encoder_attention_heads", 16) - args.speech_encoder_layers = getattr(args, "speech_encoder_layers", 6) - args.text_encoder_layers = getattr(args, "text_encoder_layers", 12) - args.encoder_shared_text_layers_from_begin = getattr( - args, "encoder_shared_text_layers_from_begin", 0 - ) - args.decoder_layers = getattr(args, "decoder_layers", 12) - args.stacked_encoder = getattr(args, "stacked_encoder", "s2s") - args.layernorm_embedding = getattr(args, "layernorm_embedding", True) - speech_text_pretrain_bart_base(args) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/models/s2t_dualinputtransformer.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/models/s2t_dualinputtransformer.py deleted file mode 100644 index e5ec336edd2ce770099364b3504cde861acc4f23..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/models/s2t_dualinputtransformer.py +++ /dev/null @@ -1,1092 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import logging -from collections import namedtuple - -import torch -import torch.nn as nn -from fairseq import checkpoint_utils, utils -from fairseq.models import ( - FairseqDecoder, - FairseqEncoder, - FairseqEncoderDecoderModel, - register_model, - register_model_architecture, -) -from fairseq.models.fairseq_encoder import EncoderOut -from fairseq.models.speech_to_text import ( - S2TTransformerEncoder, - TransformerDecoder, -) -from fairseq.models.transformer import TransformerEncoder -from fairseq.modules import ( - GradMultiply, - LayerNorm, - TransformerEncoderLayer, -) - -logger = logging.getLogger(__name__) - - -class SpeechEoSEncoder(FairseqEncoder): - def __init__(self, encoder, eos_num, feat_dim, adapter_type="None", adapter_dim=0): - super().__init__(None) - self.encoder = encoder - self.eos_num = eos_num # downsampling rate for speech input feature - self.eos_emb = ( - nn.Parameter(torch.zeros(1, feat_dim), requires_grad=True) - if eos_num > 0 - else None - ) - self.adapter = self.add_adapter(adapter_type, adapter_dim) - - def add_adapter(self, adapter_type, adapter_dim): - def _make_identity(linear, eps=1e-5): - assert isinstance(linear, nn.Linear) - linear.weight.data.mul_(eps) - linear.weight.data.fill_diagonal_(1.0) - if linear.bias is not None: - linear.bias.data.mul_(eps) - - adapter = None - if adapter_type == "Linear": - assert adapter_dim > 0 - adapter = nn.Sequential( - nn.Linear(adapter_dim, adapter_dim), LayerNorm(adapter_dim) - ) - # initialize the adapter as identity matrix first - _make_identity(adapter[0]) - - elif adapter_type == "MLP": - assert adapter_dim > 0 - # assume the model is pre-norm model - adapter = nn.Sequential( - nn.Linear(adapter_dim, 2 * adapter_dim), - nn.ReLU(), - nn.Linear(2 * adapter_dim, adapter_dim), - LayerNorm(adapter_dim), - ) - _make_identity(adapter[0]) - _make_identity(adapter[2]) - return adapter - - def add_eos(self, src_tokens, src_lengths): - bsz, max_seq_len, fdim = src_tokens.size() - if self.eos_num > 0: - src_token_eos = torch.zeros( - [bsz, max_seq_len + self.eos_num, fdim], - dtype=src_tokens.dtype, - device=src_tokens.device, - ) - src_token_eos[:, :max_seq_len] = src_tokens - for bi in range(bsz): - src_token_eos[bi][ - src_lengths[bi] : src_lengths[bi] + self.eos_num - ] = self.eos_emb.expand(self.eos_num, fdim) - src_lengths = src_lengths + self.eos_num - src_tokens = src_token_eos - return src_tokens, src_lengths - - def apply_adapter(self, enc_out): - if self.adapter is None: - return enc_out - rst = self.adapter(enc_out.encoder_out) - if enc_out.encoder_padding_mask is not None: - rst.masked_fill_( - enc_out.encoder_padding_mask.transpose(0, 1).unsqueeze(-1), 0 - ) - return EncoderOut( - encoder_out=rst, - encoder_padding_mask=enc_out.encoder_padding_mask, - encoder_embedding=enc_out.encoder_embedding, - encoder_states=enc_out.encoder_states, - src_tokens=enc_out.src_tokens, - src_lengths=enc_out.src_lengths, - ) - - def forward(self, src_tokens, src_lengths=None, return_all_hiddens=False, **kwargs): - """ - src_tokens: padded tensor (B, T, C * feat) - src_lengths: tensor of original lengths of input utterances (B,) - """ - src_tokens, src_lengths = self.add_eos(src_tokens, src_lengths) - enc_out = self.encoder(src_tokens, src_lengths, return_all_hiddens) - enc_out = self.apply_adapter(enc_out) - return enc_out - - def reorder_encoder_out(self, encoder_out, new_order): - return self.encoder.reorder_encoder_out(encoder_out, new_order) - - -class DualInputEncoder(FairseqEncoder): - def __init__( - self, - args, - spch_encoder, - text_encoder, - dictionary, - cross_attentive_loss_before_last_layer=-1, - ): - super().__init__(dictionary) - - self.spch_encoder = spch_encoder - self.text_encoder = text_encoder - self.enc_grad_mult = args.enc_grad_mult - self.cross_attentive_loss_before_last_layer = ( - cross_attentive_loss_before_last_layer - ) - self.use_cross_attentive_loss = ( - False if cross_attentive_loss_before_last_layer <= -1 else True - ) - self.enc2_along_grad_mult = args.enc2_along_grad_mult - - @classmethod - def set_shared_layer(cls, share_level, src_layer, tgt_layer): - """ - share parameters from tgt_layer to src_layer - share_level: - 0: share everything - 1: share everything but different model - 2: share weight but not bias, layernorm - """ - if share_level == 0: - return tgt_layer - if isinstance(src_layer, nn.Linear): - return tgt_layer - if isinstance(src_layer, TransformerEncoderLayer): - assert src_layer.embed_dim == tgt_layer.embed_dim - assert src_layer.normalize_before == tgt_layer.normalize_before - if share_level == 1: - src_layer.fc1 = tgt_layer.fc1 - src_layer.fc2 = tgt_layer.fc2 - src_layer.self_attn = tgt_layer.self_attn - src_layer.final_layer_norm = tgt_layer.final_layer_norm - src_layer.self_attn_layer_norm = tgt_layer.self_attn_layer_norm - src_layer.layernorm_embedding = tgt_layer.layernorm_embedding - else: - src_layer.fc1.weight = tgt_layer.fc1.weight - src_layer.fc2.weight = tgt_layer.fc2.weight - src_layer.self_attn.k_proj.weight = tgt_layer.self_attn.k_proj.weight - src_layer.self_attn.v_proj.weight = tgt_layer.self_attn.v_proj.weight - src_layer.self_attn.q_proj.weight = tgt_layer.self_attn.q_proj.weight - src_layer.self_attn.out_proj.weight = ( - tgt_layer.self_attn.out_proj.weight - ) - else: - if share_level == 1: - return tgt_layer - return src_layer - - @classmethod - def build_spch_encoder(cls, args): - cfg = { - "input_feat_per_channel": args.input_feat_per_channel, - "input_channels": args.input_channels, - "conv_kernel_sizes": args.conv_kernel_sizes, - "conv_channels": args.conv_channels, - "encoder_embed_dim": args.encoder_embed_dim, - "encoder_ffn_embed_dim": args.encoder_ffn_embed_dim, - "encoder_layers": args.speech_encoder_layers, - "encoder_layerdrop": args.encoder_layerdrop, - "encoder_attention_heads": args.encoder_attention_heads, - "max_source_positions": args.max_source_positions, - "dropout": args.dropout, - "encoder_normalize_before": args.encoder_normalize_before, - "activation_dropout": args.activation_dropout, - "attention_dropout": args.attention_dropout, - "activation_fn": args.activation_fn, - "layernorm_embedding": args.layernorm_embedding, - "no_token_positional_embeddings": args.no_token_positional_embeddings, - "no_scale_embedding": args.no_scale_embedding, - "quant_noise_pq": args.quant_noise_pq, - "encoder_freezing_updates": 0, - } - model_args = namedtuple("args", cfg.keys())(*cfg.values()) - spch_encoder = S2TTransformerEncoder(model_args) - if args.add_speech_eos: - spch_encoder = SpeechEoSEncoder( - spch_encoder, - 2 * len(args.conv_kernel_sizes.split(",")), - args.input_feat_per_channel, - adapter_type=getattr(args, "speech_encoder_adapter_type", "None"), - adapter_dim=args.encoder_embed_dim, - ) - return spch_encoder - - @classmethod - def build_text_encoder(cls, args, src_dictionary, spch_encoder): - if args.encoder_shared_layers > 0: - mx_shared_layers = ( - args.speech_encoder_layers - if args.speech_encoder_layers < args.text_encoder_layers - else args.text_encoder_layers - ) - args.encoder_shared_layers = ( - args.encoder_shared_layers - if args.encoder_shared_layers <= mx_shared_layers - else mx_shared_layers - ) - cfg = { - "encoder_embed_dim": args.encoder_text_embed_dim, - "encoder_ffn_embed_dim": args.encoder_ffn_embed_dim, - "encoder_layers": args.text_encoder_layers, - "encoder_layerdrop": args.encoder_layerdrop, - "encoder_attention_heads": args.encoder_attention_heads, - "encoder_learned_pos": args.encoder_learned_pos, - "max_source_positions": args.max_source_positions, - "dropout": args.dropout, - "encoder_normalize_before": args.encoder_normalize_before, - "activation_dropout": args.activation_dropout, - "attention_dropout": args.attention_dropout, - "activation_fn": args.activation_fn, - "adaptive_input": args.adaptive_input, - "no_token_positional_embeddings": args.no_token_positional_embeddings, - "no_scale_embedding": args.no_scale_embedding, - "quant_noise_pq": args.quant_noise_pq, - } - model_args = namedtuple("args", cfg.keys())(*cfg.values()) - enc_emb = nn.Embedding( - len(src_dictionary), model_args.encoder_embed_dim, src_dictionary.pad() - ) - text_encoder = TransformerEncoder(model_args, src_dictionary, enc_emb) - if args.add_speech_eos: - spch_encoder = spch_encoder.encoder - if args.encoder_shared_layers > 0: - text_encoder.layer_norm = cls.set_shared_layer( - args.encoder_shared_layer_level, - text_encoder.layer_norm, - spch_encoder.layer_norm, - ) - for i, ly in enumerate( - spch_encoder.transformer_layers[-args.encoder_shared_layers :] - ): - ly_id = i + args.text_encoder_layers - args.encoder_shared_layers - if not isinstance(text_encoder.layers[ly_id], type(ly)): - if text_encoder.layers[ly_id]._get_name() not in ('TransformerEncoderLayerBase', 'TransformerEncoderLayer'): - raise ValueError("The shared layers are expected from the same class") - text_encoder.layers[ly_id] = cls.set_shared_layer( - args.encoder_shared_layer_level, - text_encoder.layers[ly_id], - ly, - ) - return text_encoder - - def mult_rst_grad(self, rst, ratio): - assert isinstance(rst, dict) # instead of EncoderOut - assert len(rst["encoder_out"]) == 1 - rst["encoder_out"][0] = GradMultiply.apply(rst["encoder_out"][0], ratio) - return rst - - def process_attentive_loss_states(self, rst, interstates): - assert isinstance(rst, dict) # instead of EncoderOut - rst["encoder_states"] = interstates - return rst - - def forward( - self, - src_tokens, - src_lengths=None, - src_txt_tokens=None, - src_txt_lengths=None, - **kwargs - ): - """ - Args: - src_tokens: padded tensor (B, T, C * feat) - src_lengths: tensor of original lengths of input utterances (speech) (B,) - src_txt_tokens: padded tensor (B, T) - src_txt_lengths: tensor of original lengths of input utterances (text) (B,) - """ - # src_tokens only: inference - # src_tokens, src_lengths: speech only training - # src_txt_tokens, src_txt_lengths: text only training - # all valid: speech + text training - - if src_tokens is None and src_txt_tokens is None: - raise ValueError( - "src_tokens and src_txt_tokens cannot be None at the same time" - ) - ret1 = None - ret2 = None - return_all_hiddens = False - if src_tokens is not None: - if ( - self.use_cross_attentive_loss and src_txt_tokens is not None - ): # remove self.training so we can get attn score during validation step - return_all_hiddens = True - ret1 = self.spch_encoder( - src_tokens, src_lengths, return_all_hiddens=return_all_hiddens - ) - - if self.use_cross_attentive_loss and src_txt_tokens is not None: - assert self.cross_attentive_loss_before_last_layer < len( - ret1["encoder_states"] - ) - ret1 = self.process_attentive_loss_states( - ret1, - ret1["encoder_states"][ - -self.cross_attentive_loss_before_last_layer - 1 - ], - ) - - if src_txt_tokens is not None: - ret2 = self.text_encoder( - src_txt_tokens, src_txt_lengths, return_all_hiddens=return_all_hiddens - ) - if return_all_hiddens: - if self.cross_attentive_loss_before_last_layer == len( - self.text_encoder.layers - ): - text_embedding, _ = self.text_encoder.forward_embedding( - src_txt_tokens - ) - text_embedding = text_embedding.transpose(0, 1) - ret2 = self.process_attentive_loss_states(ret2, text_embedding) - else: - assert self.cross_attentive_loss_before_last_layer < len( - self.text_encoder.layers - ) - ret2 = self.process_attentive_loss_states( - ret2, - ret2["encoder_states"][ - -self.cross_attentive_loss_before_last_layer - 1 - ], - ) - - def merge_output(rst1, rst2): - if rst1 is None: - if not (self.enc2_along_grad_mult == 1.0 or self.training): - rst2 = self.mult_rst_grad(rst2, self.enc2_along_grad_mult) - return rst2 - if rst2 is None: - return rst1 - if self.enc_grad_mult != 1.0 and self.training: - rst1 = self.mult_rst_grad(rst1, self.enc_grad_mult) - rst2 = self.mult_rst_grad(rst2, self.enc_grad_mult) - rst = (rst1, rst2) - return rst - - return merge_output(ret1, ret2) - - def reorder_encoder_out(self, encoder_out, new_order): - assert self.training is False # used for inference only - return self.spch_encoder.reorder_encoder_out(encoder_out, new_order) - - -# TransformerMultiInputDecoder: take one or two encoder inputs -class TransformerMultiInputDecoder(FairseqDecoder): - def __init__( - self, - dictionary, - spch_decoder, - text_decoder, - compute_cross_attentive_loss=False, - cross_attentive_loss_with_norm=True, - cross_attentive_loss_reverse=False, - ): - - super().__init__(dictionary) - self.spch_decoder = spch_decoder - self.text_decoder = text_decoder - self.compute_cross_attentive_loss = compute_cross_attentive_loss - self.cross_attentive_loss_with_norm = cross_attentive_loss_with_norm - self.cross_attentive_loss_reverse = cross_attentive_loss_reverse - - @classmethod - def share_spchdecoder(cls, task_args, text_decoder, spch_decoder): - if task_args.decoder_shared_layer_level == 0: - return text_decoder - assert text_decoder.embed_tokens == spch_decoder.embed_tokens - spch_decoder.project_in_dim = text_decoder.project_in_dim - spch_decoder.embed_positions = text_decoder.embed_positions - spch_decoder.layernorm_embedding = text_decoder.layernorm_embedding - spch_decoder.project_out_dim = text_decoder.project_out_dim - spch_decoder.adaptive_softmax = text_decoder.adaptive_softmax - if task_args.decoder_shared_layer_level == 1: - spch_decoder.output_projection = text_decoder.output_projection - spch_decoder.layer_norm = text_decoder.layer_norm - else: # 2 - spch_decoder.output_projection.weight = ( - text_decoder.output_projection.weight - ) - for i, ly in enumerate(text_decoder.layers): - sly = spch_decoder.layers[i] - sly.self_attn = ly.self_attn - sly.self_attn_layer_norm = ly.self_attn_layer_norm - # sly.encoder_attn = ly.encoder_attn - if ( - task_args.decoder_shared_layer_level == 1 - ): # share everything, but under different models - sly.encoder_attn = ly.encoder_attn - sly.encoder_attn_layer_norm = ly.encoder_attn_layer_norm - sly.fc1 = ly.fc1 - sly.fc2 = ly.fc2 - sly.final_layer_norm = ly.final_layer_norm - else: # task_args.decoder_shared_layer_level == 2: #separated encoder_attn_layer_norm and bias - sly.encoder_attn.k_proj.weight = ly.encoder_attn.k_proj.weight - sly.encoder_attn.v_proj.weight = ly.encoder_attn.v_proj.weight - sly.encoder_attn.q_proj.weight = ly.encoder_attn.q_proj.weight - sly.encoder_attn.out_proj.weight = ly.encoder_attn.out_proj.weight - sly.fc1.weight = ly.fc1.weight - sly.fc2.weight = ly.fc2.weight - - return spch_decoder - - def cross_attentive_loss( - self, teacher_states, student_states, teacher_masking, student_masking, eps=1e-6 - ): - x = teacher_states.transpose(0, 1) # from T X B X D to B X T X D - y = student_states.transpose(0, 1) - if self.cross_attentive_loss_with_norm: - x = x / (x.norm(dim=2, keepdim=True) + eps) - y = y / (y.norm(dim=2, keepdim=True) + eps) - dim = x.size(-1) - # lengths: batch X seqLen - sim_scores_xy = torch.bmm(x, y.transpose(1, 2)) # batch X lenx X leny ] - if y.dtype == torch.float16: - sim_scores_xy = sim_scores_xy.float() - y = y.float() - x = x.float() - if teacher_masking != []: - assert len(teacher_masking) == 1 - sim_scores_xy = sim_scores_xy.masked_fill( - teacher_masking[0].unsqueeze(-1), float("-inf") - ) - if student_masking != []: - sim_scores_xy = sim_scores_xy.masked_fill( - student_masking[0].unsqueeze(1), float("-inf") - ) - # do masking - y_weights = utils.softmax(sim_scores_xy, dim=-1) - if teacher_masking != []: - y_weights = y_weights.masked_fill(teacher_masking[0].unsqueeze(-1), 0) - x_reconstruct_from_y = torch.bmm(y_weights, y) - - sim_scores_xx = torch.bmm(x, x.transpose(1, 2)) # batch X lenx X lenx ] - x_weights = utils.softmax(sim_scores_xx, dim=-1) - if teacher_masking != []: - x_weights = x_weights.masked_fill(teacher_masking[0].unsqueeze(-1), 0) - - # no gradient for teacher state - x_reconstruct_from_x = torch.bmm(x_weights, x).detach() - cost = (x_reconstruct_from_x - x_reconstruct_from_y).norm(dim=2) - if teacher_masking != []: - cost = cost.masked_fill(teacher_masking[0], 0) - - if not self.cross_attentive_loss_with_norm: - cost = cost / dim - return cost - - def forward( - self, - prev_output_tokens, - encoder_out, - incremental_state=None, - has_txt_input=False, - **kwargs - ): - """ - Args: - prev_output_tokens (LongTensor): previous decoder outputs of shape - `(batch, tgt_len)`, for input feeding/teacher forcing. If there are - two or more input during training, they will share the same prev_output_tokens - encoder_out (tuple[Tensor]): output from the encoder, used for - encoder-side attention. It will be tuple if there are more inputs, but a tensor - if only one input - incremental_state ([dict]): dictionary used for storing state during - :ref:`Incremental decoding`. It is only valid for inference, only from single - input - Returns: - tuple: - - the last decoder layer's output of shape `(batch, tgt_len, - vocab)`. If there are N inputs, batch will be N bigger than a single input - - the last decoder layer's attention weights of shape `(batch, - tgt_len, src_len)` - """ - assert not isinstance(encoder_out, EncoderOut) - if isinstance(encoder_out, tuple): # training with mulitple input - rst = [] - assert len(encoder_out) == 2 - for i, eo in enumerate(encoder_out): - assert incremental_state is None - if i == 0: - rst.append( - self.spch_decoder(prev_output_tokens, eo, incremental_state) - ) - else: - rst.append( - self.text_decoder(prev_output_tokens, eo, incremental_state) - ) - dec_out = torch.cat([r[0] for r in rst], dim=0) - attn_cost = None - if self.compute_cross_attentive_loss: - assert isinstance(encoder_out[0], dict) - if self.cross_attentive_loss_reverse: - attn_cost = self.cross_attentive_loss( - teacher_states=encoder_out[1]["encoder_states"], # text_states - student_states=encoder_out[0]["encoder_states"], # spch_states - teacher_masking=encoder_out[1]["encoder_padding_mask"], - student_masking=encoder_out[0]["encoder_padding_mask"], - ) - else: - attn_cost = self.cross_attentive_loss( - teacher_states=encoder_out[0]["encoder_states"], # spch_states - student_states=encoder_out[1]["encoder_states"], # text_states - teacher_masking=encoder_out[0]["encoder_padding_mask"], - student_masking=encoder_out[1]["encoder_padding_mask"], - ) - - return (dec_out, {"attn_cost": attn_cost}) - else: # inference or training with one input - if has_txt_input: - return self.text_decoder( - prev_output_tokens, encoder_out, incremental_state - ) - return self.spch_decoder(prev_output_tokens, encoder_out, incremental_state) - - -# Note: -# dual input transformer: -# encoder: S2TTransformerEncoder for speech + TransformerEncoder for text -# decoder: TransformerDecoder for text -@register_model("dual_input_s2t_transformer") -class DualInputS2TTransformerModel(FairseqEncoderDecoderModel): - def __init__(self, encoder, decoder): - super().__init__(encoder, decoder) - self.num_updates = 0 - - def max_positions(self): - return None # it is provided in task - - @staticmethod - def add_args(parser): - """Add model-specific arguments to the parser.""" - # encoder 1: S2TTransformerEncoder for speech - parser.add_argument( - "--conv-kernel-sizes", - type=str, - metavar="N", - help="kernel sizes of Conv1d subsampling layers", - ) - parser.add_argument( - "--conv-channels", - type=int, - metavar="N", - help="# of channels in Conv1d subsampling layers", - ) - parser.add_argument( - "--enc-output-dim", - type=int, - metavar="N", - help=""" - encoder output dimension, can be None. If specified, projecting the - transformer output to the specified dimension""", - ) - # standard Transformer - parser.add_argument( - "--activation-fn", - type=str, - default="relu", - choices=utils.get_available_activation_fns(), - help="activation function to use", - ) - parser.add_argument( - "--dropout", type=float, metavar="D", help="dropout probability" - ) - parser.add_argument( - "--attention-dropout", - type=float, - metavar="D", - help="dropout probability for attention weights", - ) - parser.add_argument( - "--activation-dropout", - "--relu-dropout", - type=float, - metavar="D", - help="dropout probability after activation in FFN.", - ) - parser.add_argument( - "--encoder-embed-dim", - type=int, - metavar="N", - help="encoder embedding dimension", - ) - parser.add_argument( - "--encoder-text-embed-dim", - type=int, - metavar="N", - help="encoder text embedding dimension", - ) - parser.add_argument( - "--encoder-ffn-embed-dim", - type=int, - metavar="N", - help="encoder embedding dimension for FFN", - ) - parser.add_argument( - "--encoder-attention-heads", - type=int, - metavar="N", - help="num encoder attention heads", - ) - parser.add_argument( - "--decoder-embed-dim", - type=int, - metavar="N", - help="decoder embedding dimension", - ) - parser.add_argument( - "--decoder-ffn-embed-dim", - type=int, - metavar="N", - help="decoder embedding dimension for FFN", - ) - parser.add_argument( - "--decoder-layers", type=int, metavar="N", help="num decoder layers" - ) - parser.add_argument( - "--decoder-attention-heads", - type=int, - metavar="N", - help="num decoder attention heads", - ) - parser.add_argument( - "--layernorm-embedding", - action="store_true", - help="add layernorm to embedding", - ) - parser.add_argument( - "--no-scale-embedding", - action="store_true", - help="if True, dont scale embeddings", - ) - # non-standard transformer parameters - parser.add_argument( - "--speech-encoder-layers", - type=int, - metavar="N", - help="num speech encoder layers", - ) - parser.add_argument( - "--text-encoder-layers", - type=int, - metavar="N", - help="num text encoder layers", - ) - parser.add_argument( - "--encoder-shared-layers", - type=int, - metavar="N", - help="num shared encoder layers", - ) - parser.add_argument( - "--encoder-shared-layer-level", - type=int, - metavar="N", - default=0, - choices=[0, 1, 2], - help="share layer level 0: all share 1: all share with separate model 2: share weight but not bias and layernorm", - ) - - parser.add_argument( - "--decoder-shared-layer-level", - default=0, - choices=[0, 1, 2], - type=int, - metavar="N", - help="0: share everything; 1: share everything with different model 2: no share layer_norm and bias", - ) - ### - parser.add_argument( - "--text-input-cost-ratio", - type=float, - default=1.0, - metavar="V", - help="text input cost ratio relative to speech input cost", - ) - parser.add_argument( - "--init-scale", - type=float, - default=1.0, - metavar="V", - help="scale the initial weight by given factor", - ) - parser.add_argument( - "--enc-grad-mult", - type=float, - metavar="V", - default=1.0, - help="multiply enc1 and enc2 gradient by V", - ) - parser.add_argument( - "--enc2-along-grad-mult", - type=float, - metavar="V", - default=1.0, - help="multiply enc2 gradient by V if only enc2 is used", - ) - parser.add_argument( - "--load-pretrain-encoder", - type=str, - default="", - metavar="EXPR", - help=""" path to the pretrained encoder """, - ) - parser.add_argument( - "--load-pretrain-speech-encoder", - type=str, - default="", - metavar="EXPR", - help=""" path to the pretrained speech encoder """, - ) - parser.add_argument( - "--load-pretrain-text-encoder", - type=str, - default="", - metavar="EXPR", - help=""" path to the pretrained text encoder """, - ) - parser.add_argument( - "--load-pretrain-text-encoder-last", - type=str, - default="", - metavar="EXPR", - help=""" path to the pretrained text encoder """, - ) - parser.add_argument( - "--load-pretrain-decoder", - type=str, - metavar="EXPR", - default="", - help=""" path to the pretrained encoder """, - ) - parser.add_argument( - "--add-speech-eos", - action="store_true", - help="add eos token at the end of input feature", - ) - parser.add_argument( - "--speech-encoder-adapter-type", - type=str, - metavar="EXPR", - default="None", - choices=["None", "Linear", "MLP"], - help="add speech encoder adapter", - ) - - @classmethod - def build_encoder(cls, args, task): - spch_encoder = DualInputEncoder.build_spch_encoder(args) - text_encoder = DualInputEncoder.build_text_encoder( - args, task.src_dict, spch_encoder - ) - cross_attentive_loss_before_last_layer = ( - 0 if getattr(args, "attentive_cost_regularization", 0.0) > 0.0 else -1 - ) - encoder = DualInputEncoder( - args, - spch_encoder, - text_encoder, - task.src_dict, - cross_attentive_loss_before_last_layer, - ) - if args.init_scale != 1.0: - with torch.no_grad(): - for param in encoder.parameters(): - param.data.mul_(args.init_scale) - if args.load_pretrain_text_encoder != "": - checkpoint_utils.load_pretrained_component_from_model( - text_encoder, args.load_pretrain_text_encoder - ) - if args.load_pretrain_speech_encoder != "": - if hasattr(spch_encoder, "encoder"): - checkpoint_utils.load_pretrained_component_from_model( - spch_encoder.encoder, args.load_pretrain_speech_encoder - ) - else: - checkpoint_utils.load_pretrained_component_from_model( - spch_encoder, args.load_pretrain_speech_encoder - ) - if ( - args.load_pretrain_text_encoder_last != "" - ): # if share encoder, speech encoder parameters will be used. - # It provides a chance to use pre-trained mt encoder instead - checkpoint_utils.load_pretrained_component_from_model( - text_encoder, args.load_pretrain_text_encoder_last - ) - - if args.load_pretrain_encoder != "": - checkpoint_utils.load_pretrained_component_from_model( - encoder, args.load_pretrain_encoder - ) - return encoder - - @classmethod - def build_decoder(cls, args, task): - dec_cfg = { - "decoder_layerdrop": args.decoder_layerdrop, - "share_decoder_input_output_embed": args.share_decoder_input_output_embed, - "decoder_embed_dim": args.decoder_embed_dim, - "max_target_positions": args.max_target_positions, - "dropout": args.dropout, - "encoder_learned_pos": args.encoder_learned_pos, - "decoder_learned_pos": args.decoder_learned_pos, - "layernorm_embedding": args.layernorm_embedding, - "decoder_normalize_before": args.decoder_normalize_before, - "activation_dropout": args.activation_dropout, - "attention_dropout": args.attention_dropout, - "decoder_ffn_embed_dim": args.decoder_ffn_embed_dim, - "decoder_layers": args.decoder_layers, - "decoder_attention_heads": args.decoder_attention_heads, - "decoder_output_dim": args.decoder_embed_dim, - "no_scale_embedding": args.no_scale_embedding, - "adaptive_input": args.adaptive_input, - "quant_noise_pq": args.quant_noise_pq, - "adaptive_softmax_cutoff": args.adaptive_softmax_cutoff, - "tie_adaptive_weights": args.tie_adaptive_weights, - "no_token_positional_embeddings": args.no_token_positional_embeddings, - "encoder": {"embed_dim":args.encoder_embed_dim} - } - dec_cfg = namedtuple("args", dec_cfg.keys())(*dec_cfg.values()) - dec_emb = nn.Embedding( - len(task.target_dictionary), - args.decoder_embed_dim, - task.target_dictionary.pad(), - ) - compute_cross_attentive_loss = ( - True if getattr(args, "attentive_cost_regularization", 0.0) > 0.0 else False - ) - cross_attentive_loss_without_norm = getattr( - args, "attentive_cost_without_normalize", False - ) - cross_attentive_loss_reverse = ( - False # getattr(args, "attentive_cost_reverse", False) - ) - - text_decoder = TransformerDecoder(dec_cfg, task.target_dictionary, dec_emb) - spch_decoder = TransformerDecoder(dec_cfg, task.target_dictionary, dec_emb) - spch_decoder = TransformerMultiInputDecoder.share_spchdecoder( - args, text_decoder, spch_decoder - ) - decoder = TransformerMultiInputDecoder( - dictionary=task.target_dictionary, - spch_decoder=spch_decoder, - text_decoder=text_decoder, - compute_cross_attentive_loss=compute_cross_attentive_loss, - cross_attentive_loss_with_norm=True - if not cross_attentive_loss_without_norm - else False, - cross_attentive_loss_reverse=cross_attentive_loss_reverse, - ) - if args.init_scale != 1.0: - with torch.no_grad(): - for param in decoder.parameters(): - param.data.mul_(args.init_scale) - if args.load_pretrain_decoder != "": - try: - checkpoint_utils.load_pretrained_component_from_model( - decoder, args.load_pretrain_decoder - ) - except RuntimeError: - checkpoint_utils.load_pretrained_component_from_model( - decoder.text_decoder, args.load_pretrain_decoder - ) - if args.decoder_shared_layer_level > 0: - checkpoint_utils.load_pretrained_component_from_model( - decoder.spch_decoder, args.load_pretrain_decoder - ) - - return decoder - - @classmethod - def build_model(cls, args, task): - """Build a new model instance.""" - # make sure that all args are properly defaulted - # (in case there are any new ones) - dualinputs2ttransformer_base(args) - - encoder = cls.build_encoder(args, task) - decoder = cls.build_decoder(args, task) - return cls(encoder, decoder) - - def get_normalized_probs(self, net_output, log_probs, sample=None): - # net_output['encoder_out'] is a (B, T, D) tensor - lprobs = super().get_normalized_probs(net_output, log_probs, sample) - lprobs.batch_first = True - return lprobs - - def set_num_updates(self, num_updates): - """Set the number of parameters updates.""" - super().set_num_updates(num_updates) - self.num_updates = num_updates - - def forward( - self, - src_tokens, - src_lengths, - prev_output_tokens, - use_encoder_outputs=False, - src_txt_tokens=None, - src_txt_lengths=None, - mode="sup_speech", - **kwargs - ): - """ - Run the forward pass for an encoder-decoder model. - - First feed a batch of source tokens through the encoder. Then, feed the - encoder output and previous decoder outputs (i.e., teacher forcing) to - the decoder to produce the next outputs:: - - encoder_out = self.encoder(src_tokens, src_lengths) - return self.decoder(prev_output_tokens, encoder_out) - - Args: - src_tokens (LongTensor): tokens in the source language of shape - `(batch, src_len)` - src_lengths (LongTensor): source sentence lengths of shape `(batch)` - prev_output_tokens (LongTensor): previous decoder outputs of shape - `(batch, tgt_len)`, for teacher forcing - mode = 'sup_speech' or 'text' - - Returns: - tuple: - - the decoder's output of shape `(batch, tgt_len, vocab)` - - a dictionary with any model-specific outputs - """ - if mode == "text": - assert src_txt_tokens is None - src_txt_tokens = src_tokens - src_txt_lengths = src_lengths - src_tokens = None - src_lengths = None - encoder_out = self.encoder( - src_tokens, - src_lengths=src_lengths, - src_txt_tokens=src_txt_tokens, - src_txt_lengths=src_txt_lengths, - **kwargs - ) - has_txt_input = True if src_txt_tokens is not None else False - decoder_out = self.decoder( - prev_output_tokens, - encoder_out=encoder_out, - has_txt_input=has_txt_input, - **kwargs - ) - if use_encoder_outputs: - return decoder_out, encoder_out - return decoder_out - - -@register_model_architecture( - "dual_input_s2t_transformer", "dualinputs2ttransformer_base" -) -def dualinputs2ttransformer_base(args): - args.encoder_freezing_updates = getattr(args, "encoder_freezing_updates", 0) - # Convolutional subsampler - args.input_feat_per_channel = getattr(args, "input_feat_per_channel", 80) - args.conv_kernel_sizes = getattr(args, "conv_kernel_sizes", "5,5") - args.conv_channels = getattr(args, "conv_channels", 1024) - # Transformer - args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 512) - args.encoder_text_embed_dim = getattr( - args, "encoder_text_embed_dim", args.encoder_embed_dim - ) - args.encoder_ffn_embed_dim = getattr(args, "encoder_ffn_embed_dim", 2048) - args.encoder_attention_heads = getattr(args, "encoder_attention_heads", 8) - args.encoder_normalize_before = getattr(args, "encoder_normalize_before", True) - args.encoder_layerdrop = getattr(args, "encoder_layerdrop", 0) - args.encoder_learned_pos = getattr(args, "encoder_learned_pos", False) - - args.decoder_embed_dim = getattr(args, "decoder_embed_dim", args.encoder_embed_dim) - args.decoder_ffn_embed_dim = getattr( - args, "decoder_ffn_embed_dim", args.encoder_ffn_embed_dim - ) - args.decoder_attention_heads = getattr(args, "decoder_attention_heads", 8) - args.decoder_normalize_before = getattr(args, "decoder_normalize_before", True) - args.decoder_learned_pos = getattr(args, "decoder_learned_pos", False) - args.dropout = getattr(args, "dropout", 0.1) - args.attention_dropout = getattr(args, "attention_dropout", args.dropout) - args.activation_dropout = getattr(args, "activation_dropout", args.dropout) - args.activation_fn = getattr(args, "activation_fn", "relu") - args.adaptive_softmax_cutoff = getattr(args, "adaptive_softmax_cutoff", None) - args.adaptive_softmax_dropout = getattr(args, "adaptive_softmax_dropout", 0) - args.tie_adaptive_weights = getattr(args, "tie_adaptive_weights", False) - args.share_decoder_input_output_embed = getattr( - args, "share_decoder_input_output_embed", False - ) - args.no_token_positional_embeddings = getattr( - args, "no_token_positional_embeddings", False - ) - args.adaptive_input = getattr(args, "adaptive_input", False) - args.decoder_layerdrop = getattr(args, "decoder_layerdrop", 0.0) - args.decoder_output_dim = getattr( - args, "decoder_output_dim", args.decoder_embed_dim - ) - args.layernorm_embedding = getattr(args, "layernorm_embedding", False) - args.no_scale_embedding = getattr(args, "no_scale_embedding", False) - args.quant_noise_pq = getattr(args, "quant_noise_pq", 0) - - args.speech_encoder_layers = getattr(args, "speech_encoder_layers", 10) - args.text_encoder_layers = getattr(args, "text_encoder_layers", 6) - args.encoder_shared_layers = getattr(args, "encoder_shared_layers", 0) - args.decoder_layers = getattr(args, "decoder_layers", 6) - - args.add_speech_eos = getattr(args, "add_speech_eos", False) - - -@register_model_architecture("dual_input_s2t_transformer", "dualinputs2ttransformer_s") -def dualinputs2ttransformer_s(args): - args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 256) - args.encoder_ffn_embed_dim = getattr(args, "encoder_ffn_embed_dim", 256 * 4) - args.encoder_attention_heads = getattr(args, "encoder_attention_heads", 4) - args.decoder_attention_heads = getattr(args, "decoder_attention_heads", 4) - args.dropout = getattr(args, "dropout", 0.1) - args.speech_encoder_layers = getattr(args, "speech_encoder_layers", 7) - args.text_encoder_layers = getattr(args, "text_encoder_layers", 7) - args.decoder_layers = getattr(args, "decoder_layers", 7) - dualinputs2ttransformer_base(args) - - -@register_model_architecture("dual_input_s2t_transformer", "dualinputs2ttransformer_m") -def dualinputs2ttransformer_m(args): - args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 512) - args.encoder_ffn_embed_dim = getattr(args, "encoder_ffn_embed_dim", 512 * 4) - args.encoder_attention_heads = getattr(args, "encoder_attention_heads", 8) - args.decoder_attention_heads = getattr(args, "decoder_attention_heads", 8) - args.dropout = getattr(args, "dropout", 0.15) - args.speech_encoder_layers = getattr(args, "speech_encoder_layers", 10) - args.text_encoder_layers = getattr(args, "text_encoder_layers", 6) - args.decoder_layers = getattr(args, "decoder_layers", 6) - dualinputs2ttransformer_base(args) - - -@register_model_architecture("dual_input_s2t_transformer", "dualinputs2ttransformer_b") -def dualinputs2ttransformer_b(args): - args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 768) - args.encoder_ffn_embed_dim = getattr(args, "encoder_ffn_embed_dim", 768 * 4) - args.encoder_attention_heads = getattr(args, "encoder_attention_heads", 12) - args.decoder_attention_heads = getattr(args, "decoder_attention_heads", 12) - args.dropout = getattr(args, "dropout", 0.15) - args.speech_encoder_layers = getattr(args, "speech_encoder_layers", 12) - args.text_encoder_layers = getattr(args, "text_encoder_layers", 6) - args.decoder_layers = getattr(args, "decoder_layers", 6) - dualinputs2ttransformer_base(args) - - -@register_model_architecture("dual_input_s2t_transformer", "dualinputs2ttransformer_l") -def dualinputs2ttransformer_l(args): - args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 1024) - args.encoder_ffn_embed_dim = getattr(args, "encoder_ffn_embed_dim", 1024 * 4) - args.encoder_attention_heads = getattr(args, "encoder_attention_heads", 16) - args.decoder_attention_heads = getattr(args, "decoder_attention_heads", 16) - args.dropout = getattr(args, "dropout", 0.2) - args.speech_encoder_layers = getattr(args, "speech_encoder_layers", 12) - args.text_encoder_layers = getattr(args, "text_encoder_layers", 6) - args.decoder_layers = getattr(args, "decoder_layers", 6) - dualinputs2ttransformer_base(args) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/models/s2t_dualinputwavtransformer.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/models/s2t_dualinputwavtransformer.py deleted file mode 100644 index 165c53d58976ccf91186fc3d661dcb88eefa925d..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/models/s2t_dualinputwavtransformer.py +++ /dev/null @@ -1,525 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import logging -from collections import OrderedDict, namedtuple - -import torch.nn as nn -from fairseq import checkpoint_utils, utils -from fairseq.checkpoint_utils import load_checkpoint_to_cpu -from fairseq.file_io import PathManager -from fairseq.models import register_model, register_model_architecture -from fairseq.models.speech_to_text import ( - SpeechWavTransformerEncoder, - StackedSpeechWavTransformerEncoder, - TransformerDecoder, -) -from fairseq.models.transformer import TransformerEncoder - -from .s2t_dualinputtransformer import ( - DualInputEncoder, - DualInputS2TTransformerModel, - TransformerMultiInputDecoder, -) - -logger = logging.getLogger(__name__) - - -@register_model("dual_input_wav_transformer") -class DualInputWavTransformerModel(DualInputS2TTransformerModel): - def __init__(self, encoder, decoder): - super().__init__(encoder, decoder) - - @staticmethod - def add_args(parser): - def add_transformer_args(parser): - # We can't use TransformerModel.add_args(parser), since it defines max-source-positions which is duplicated with tasks/speech_to_text.py - # Transformer - parser.add_argument( - "--activation-fn", - type=str, - default="relu", - choices=utils.get_available_activation_fns(), - help="activation function to use", - ) - parser.add_argument( - "--dropout", type=float, metavar="D", help="dropout probability" - ) - parser.add_argument( - "--attention-dropout", - type=float, - metavar="D", - help="dropout probability for attention weights", - ) - parser.add_argument( - "--activation-dropout", - "--relu-dropout", - type=float, - metavar="D", - help="dropout probability after activation in FFN.", - ) - parser.add_argument( - "--encoder-embed-dim", - type=int, - metavar="N", - help="encoder embedding dimension", - ) - parser.add_argument( - "--encoder-ffn-embed-dim", - type=int, - metavar="N", - help="encoder embedding dimension for FFN", - ) - parser.add_argument( - "--encoder-layers", type=int, metavar="N", help="num encoder layers" - ) - parser.add_argument( - "--encoder-attention-heads", - type=int, - metavar="N", - help="num encoder attention heads", - ) - parser.add_argument( - "--encoder-normalize-before", - action="store_true", - help="apply layernorm before each encoder block", - ) - parser.add_argument( - "--decoder-embed-dim", - type=int, - metavar="N", - help="decoder embedding dimension", - ) - parser.add_argument( - "--decoder-ffn-embed-dim", - type=int, - metavar="N", - help="decoder embedding dimension for FFN", - ) - parser.add_argument( - "--decoder-layers", type=int, metavar="N", help="num decoder layers" - ) - parser.add_argument( - "--decoder-attention-heads", - type=int, - metavar="N", - help="num decoder attention heads", - ) - parser.add_argument( - "--decoder-normalize-before", - action="store_true", - help="apply layernorm before each decoder block", - ) - parser.add_argument( - "--share-decoder-input-output-embed", - action="store_true", - help="share decoder input and output embeddings", - ) - parser.add_argument( - "--layernorm-embedding", - action="store_true", - help="add layernorm to embedding", - ) - parser.add_argument( - "--no-scale-embedding", - action="store_true", - help="if True, dont scale embeddings", - ) - - parser.add_argument( - "--encoder-learned-pos", - action="store_true", - help="use learned positional embeddings", - ) - parser.add_argument( - "--decoder-learned-pos", - action="store_true", - help="use learned positional embeddings", - ) - - add_transformer_args(parser) - SpeechWavTransformerEncoder.add_args(parser) - parser.add_argument( - "--load-pretrained-speech-text-encoder", - type=str, - default="", - metavar="EXPR", - help=""" path to the pretrained speech text encoder from SpeechTextPreTrainModel """, - ) - parser.add_argument( - "--load-pretrained-wav2vec-encoder", - type=str, - default="", - metavar="EXPR", - help=""" path to the pretrained speech text encoder from wav2vec """, - ) - - parser.add_argument( - "--load-pretrained-speech-text-decoder", - type=str, - default="", - metavar="EXPR", - help=""" path to the pretrained speech text decoder from SpeechTextPreTrainModel """, - ) - parser.add_argument( - "--load-pretrained-text-decoder", - type=str, - default="", - metavar="EXPR", - help=""" path to the pretrained text decoder """, - ) - parser.add_argument( - "--load-init-encoder", - type=str, - default="", - metavar="EXPR", - help=""" path to load seed encoder model """, - ) - parser.add_argument( - "--load-init-decoder", - type=str, - default="", - metavar="EXPR", - help=""" path to load seed decoder model """, - ) - - parser.add_argument( - "--text-input-cost-ratio", - type=float, - default=1.0, - metavar="V", - help="text input cost ratio relative to speech input cost", - ) - parser.add_argument( - "--enc-grad-mult", - type=float, - metavar="V", - default=1.0, - help="multiply enc1 and enc2 gradient by V", - ) - parser.add_argument( - "--enc2-along-grad-mult", - type=float, - metavar="V", - default=1.0, - help="multiply enc2 gradient by V if only enc2 is used", - ) - parser.add_argument( - "--no-strict-check-pretrain-model", - action="store_true", - help="Don't apply strict model check for the pretrained model", - ) - - parser.add_argument( - "--stacked-encoder", - action="store_true", - help="stack speech and text encoders", - ) - - @classmethod - def update_transformer_encoder_cfg(cls, args, update_dict): - cfg = dict(args._get_kwargs()) - for fkey in update_dict.keys(): - cfg[fkey] = update_dict[fkey] - cfg.pop("_name", None) # remove keys start with _ - model_args = namedtuple("args", cfg.keys())(*cfg.values()) - return model_args - - @classmethod - def build_text_encoder(cls, args, src_dictionary): - enc_emb = nn.Embedding( - len(src_dictionary), args.encoder_embed_dim, src_dictionary.pad() - ) - model_args = cls.update_transformer_encoder_cfg( - args, - { - "encoder_layers": args.text_encoder_layers, - "max_source_positions": args.max_positions_text, - }, - ) - text_encoder = TransformerEncoder(model_args, src_dictionary, enc_emb) - return text_encoder - - @classmethod - def build_speech_encoder(cls, args): - model_args = cls.update_transformer_encoder_cfg( - args, {"encoder_layers": args.speech_encoder_layers} - ) - speech_encoder = SpeechWavTransformerEncoder(model_args) - return speech_encoder - - @classmethod - def check_args(cls, condition, is_strict, msg): - if condition: - return - if is_strict: - raise ValueError(msg) - logger.warn(msg) - - @classmethod - def build_encoder(cls, args, task): - # text_encoder = cls.build_text_encoder(args, task.source_dictionary ) - text_encoder = cls.build_text_encoder(args, task.src_dict) - speech_encoder = cls.build_speech_encoder(args) - if args.load_pretrained_wav2vec_encoder: - component_pairs = ( - ("feature_extractor", speech_encoder.subsample), - ("post_extract_proj", speech_encoder.feat_proj), - ("layer_norm", speech_encoder.feat_layer_norm), - ("encoder.pos_conv", speech_encoder.embed_positions), - ("encoder.layers", speech_encoder.layers), - ("encoder.layer_norm", speech_encoder.layer_norm), - ("mask_emb", speech_encoder.mask_emb), - ) - state = cls.load_pretrained_speech_text_components( - args.load_pretrained_wav2vec_encoder, component_pairs - ) - cls.check_args( - args.encoder_normalize_before - == state["cfg"]["model"]["layer_norm_first"], - not args.no_strict_check_pretrain_model, - f"encoder_normalize_before {args.encoder_normalize_before} doesn't match with the pretrained model", - ) - cls.check_args( - args.activation_fn == state["cfg"]["model"]["activation_fn"], - not args.no_strict_check_pretrain_model, - f"activation_fn {args.activation_fn} doesn't match with the pretrained model", - ) - - if getattr(args, "stacked_encoder", False): - if args.encoder_shared_text_layers_from_begin > 0: - raise ValueError( - "We can not stack encoders and share encoders at the same time!" - ) - speech_encoder = StackedSpeechWavTransformerEncoder( - speech_encoder, text_encoder.layers, text_encoder.layer_norm - ) - else: - cls.share_speech_text_encoder( - speech_encoder, text_encoder, args.encoder_shared_text_layers_from_begin - ) - - cross_attentive_loss_before_last_layer = ( - 0 if getattr(args, "attentive_cost_regularization", 0.0) > 0.0 else -1 - ) - encoder = DualInputEncoder( - args, - speech_encoder, - text_encoder, - task.src_dict, - cross_attentive_loss_before_last_layer, - ) - if args.load_pretrained_speech_text_encoder: - component_pairs = ( - ("encoder.sup_s2s_speech_encoder", encoder.spch_encoder), - ("encoder.text_encoder", encoder.text_encoder), - ) - cls.load_pretrained_speech_text_components( - args.load_pretrained_speech_text_encoder, component_pairs - ) - if getattr(args, "load_init_encoder", "") != "": - checkpoint_utils.load_pretrained_component_from_model( - encoder, args.load_init_encoder - ) - return encoder - - @classmethod - def build_text_decoder(cls, args, tgt_dictionary, dec_emb_share=None): - dec_emb = ( - nn.Embedding( - len(tgt_dictionary), args.decoder_embed_dim, tgt_dictionary.pad() - ) - if dec_emb_share is None - else dec_emb_share - ) - text_decoder = TransformerDecoder(args, tgt_dictionary, dec_emb) - return text_decoder - - @classmethod - def build_decoder(cls, args, task): - text_decoder = cls.build_text_decoder(args, task.target_dictionary) - compute_cross_attentive_loss = ( - True if getattr(args, "attentive_cost_regularization", 0.0) > 0.0 else False - ) - cross_attentive_loss_without_norm = getattr( - args, "attentive_cost_without_normalize", False - ) - cross_attentive_loss_reverse = ( - False # getattr(args, "attentive_cost_reverse", False) - ) - if getattr(args, "load_pretrained_text_decoder", "") != "": - checkpoint_utils.load_pretrained_component_from_model( - text_decoder, args.load_pretrained_text_decoder - ) - - if args.load_pretrained_speech_text_decoder: - component_pairs = (("decoder.text_decoder", text_decoder),) - cls.load_pretrained_speech_text_components( - args.load_pretrained_speech_text_decoder, component_pairs - ) - - decoder = TransformerMultiInputDecoder( - dictionary=task.target_dictionary, - spch_decoder=text_decoder, - text_decoder=text_decoder, - compute_cross_attentive_loss=compute_cross_attentive_loss, - cross_attentive_loss_with_norm=True - if not cross_attentive_loss_without_norm - else False, - cross_attentive_loss_reverse=cross_attentive_loss_reverse, - ) - if getattr(args, "load_init_decoder", "") != "": - checkpoint_utils.load_pretrained_component_from_model( - decoder, args.load_init_decoder - ) - return decoder - - @classmethod - def load_pretrained_speech_text_components(cls, checkpoint, component_pairs): - if not PathManager.exists(checkpoint): - raise IOError("Model file not found: {}".format(checkpoint)) - state = load_checkpoint_to_cpu(checkpoint) - for component_type, component in component_pairs: - if isinstance(component, nn.parameter.Parameter): - component.data.copy_(state["model"][component_type]) - else: - component_state_dict = OrderedDict() - for key in state["model"].keys(): - if key.startswith(component_type): - component_subkey = key[len(component_type) + 1 :] - component_state_dict[component_subkey] = state["model"][key] - component.load_state_dict(component_state_dict, strict=True) - return state - - @classmethod - def share_speech_text_encoder( - cls, speech_encoder, text_encoder, shared_layers_from_begin - ): - if shared_layers_from_begin > 0: - num_text_encoder_layers = len(text_encoder.layers) - assert len(speech_encoder.layers) >= shared_layers_from_begin - assert num_text_encoder_layers >= shared_layers_from_begin - assert len(speech_encoder.layers) >= num_text_encoder_layers - for i, ly in enumerate( - speech_encoder.layers[ - -num_text_encoder_layers : -num_text_encoder_layers - + shared_layers_from_begin - ] - ): - assert isinstance(text_encoder.layers[i], type(ly)) - text_encoder.layers[i] = ly - - -@register_model_architecture( - "dual_input_wav_transformer", "dualinputs2twavtransformer_base" -) -def dualinputs2twavtransformer_base(args): - # speech masking - args.dropout_input = getattr(args, "dropout_input", 0) - args.dropout_features = getattr(args, "dropout_features", 0) - args.speech_mask_length = getattr(args, "speech_mask_length", 10) - args.speech_mask_prob = getattr(args, "speech_mask_prob", 0.65) - args.speech_mask_selection = getattr(args, "speech_mask_selection", "static") - args.speech_mask_other = getattr(args, "speech_mask_other", 0) - args.speech_mask_min_space = getattr(args, "speech_mask_min_space", 1) - args.speech_no_mask_overlap = getattr(args, "speech_no_mask_overlap", False) - args.speech_conv_bias = getattr(args, "speech_conv_bias", False) - args.speech_extractor_mode = getattr(args, "speech_extractor_mode", "default") - args.no_strict_check_pretrain_model = getattr( - args, "no_strict_check_pretrain_model", False - ) - - args.speech_mask_channel_length = getattr(args, "speech_mask_channel_length", 10) - args.speech_mask_channel_prob = getattr(args, "speech_mask_channel_prob", 0.0) - args.speech_mask_channel_selection = getattr( - args, "speech_mask_channel_selection", "static" - ) - args.speech_mask_channel_other = getattr(args, "speech_mask_channel_other", 0) - args.speech_mask_channel_min_space = getattr( - args, "speech_mask_channel_min_space", 1 - ) - args.speech_no_mask_channel_overlap = getattr( - args, "speech_no_mask_channel_overlap", False - ) - args.no_scale_feature = getattr(args, "", False) - args.feature_grad_mult = getattr(args, "feature_grad_mult", 0.0) # 0.1 - - # Transformer - args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 768) - args.encoder_ffn_embed_dim = getattr( - args, "encoder_ffn_embed_dim", args.encoder_embed_dim * 4 - ) - args.encoder_attention_heads = getattr(args, "encoder_attention_heads", 12) - args.encoder_normalize_before = getattr(args, "encoder_normalize_before", False) - args.encoder_layerdrop = getattr(args, "encoder_layerdrop", 0.1) - args.encoder_learned_pos = getattr(args, "encoder_learned_pos", False) - - args.decoder_embed_dim = getattr(args, "decoder_embed_dim", args.encoder_embed_dim) - args.decoder_ffn_embed_dim = getattr( - args, "decoder_ffn_embed_dim", args.encoder_ffn_embed_dim - ) - args.decoder_attention_heads = getattr( - args, "decoder_attention_heads", args.encoder_attention_heads - ) - args.decoder_normalize_before = getattr(args, "decoder_normalize_before", False) - args.decoder_learned_pos = getattr(args, "decoder_learned_pos", False) - args.dropout = getattr(args, "dropout", 0.1) - args.attention_dropout = getattr(args, "attention_dropout", 0) - args.activation_dropout = getattr(args, "activation_dropout", args.dropout) - args.activation_fn = getattr(args, "activation_fn", "relu") # gelu? - args.adaptive_softmax_cutoff = getattr(args, "adaptive_softmax_cutoff", None) - args.adaptive_softmax_dropout = getattr(args, "adaptive_softmax_dropout", 0) - args.tie_adaptive_weights = getattr(args, "tie_adaptive_weights", False) - args.share_decoder_input_output_embed = getattr( - args, "share_decoder_input_output_embed", False - ) - args.no_token_positional_embeddings = getattr( - args, "no_token_positional_embeddings", False - ) - args.adaptive_input = getattr(args, "adaptive_input", False) - args.decoder_layerdrop = getattr(args, "decoder_layerdrop", 0.0) - args.decoder_output_dim = getattr( - args, "decoder_output_dim", args.decoder_embed_dim - ) - args.layernorm_embedding = getattr(args, "layernorm_embedding", False) - args.no_scale_embedding = getattr(args, "no_scale_embedding", False) - args.quant_noise_pq = getattr(args, "quant_noise_pq", 0) - - args.speech_encoder_layers = getattr(args, "speech_encoder_layers", 12) - args.text_encoder_layers = getattr(args, "text_encoder_layers", 6) - args.encoder_shared_text_layers_from_begin = getattr( - args, "encoder_shared_text_layers_from_begin", 6 - ) - args.decoder_layers = getattr(args, "decoder_layers", 6) - - -@register_model_architecture( - "dual_input_wav_transformer", "dualinputs2twavtransformer_base_stack" -) -def dualinputs2twavtransformer_base_stack(args): - args.speech_encoder_layers = getattr(args, "speech_encoder_layers", 6) - args.text_encoder_layers = getattr(args, "text_encoder_layers", 6) - args.encoder_shared_text_layers_from_begin = getattr( - args, "encoder_shared_text_layers_from_begin", 0 - ) - args.decoder_layers = getattr(args, "decoder_layers", 6) - args.stacked_encoder = getattr(args, "stacked_encoder", True) - args.layernorm_embedding = getattr(args, "layernorm_embedding", True) - dualinputs2twavtransformer_base(args) - - -@register_model_architecture( - "dual_input_wav_transformer", "dualinputs2twavtransformer_large" -) -def dualinputs2twavtransformer_large(args): - args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 1024) - args.encoder_attention_heads = getattr(args, "encoder_attention_heads", 16) - args.speech_encoder_layers = getattr(args, "speech_encoder_layers", 24) - args.text_encoder_layers = getattr(args, "text_encoder_layers", 12) - args.encoder_shared_text_layers_from_begin = getattr( - args, "encoder_shared_text_layers_from_begin", 12 - ) - args.decoder_layers = getattr(args, "decoder_layers", 12) - dualinputs2twavtransformer_base(args) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/models/s2t_dualinputxmtransformer.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/models/s2t_dualinputxmtransformer.py deleted file mode 100644 index a9a70384a742ceca1d55874d446e2af7cc62048e..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/models/s2t_dualinputxmtransformer.py +++ /dev/null @@ -1,583 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import copy - -import torch.nn as nn -from fairseq import checkpoint_utils, utils -from fairseq.data.data_utils import lengths_to_padding_mask -from fairseq.models import ( - FairseqEncoder, - register_model, - register_model_architecture, -) -from fairseq.models.speech_to_text import Wav2VecEncoderWithAdaptor -from fairseq.models.speech_to_text.xm_transformer import ( - need_finetuning, - set_default_adaptor_args, - set_default_w2v_encoder_args, -) -from fairseq.models.transformer import TransformerDecoder, TransformerEncoder -from fairseq.models.wav2vec import TransformerSentenceEncoderLayer -from fairseq.utils import safe_hasattr - -from .s2t_dualinputtransformer import ( - DualInputEncoder, - DualInputS2TTransformerModel, - TransformerMultiInputDecoder, -) - - -class TransformerSentenceEncoderLayerStd(TransformerSentenceEncoderLayer): - def __init__(self, sent_enc_layer): - super(TransformerSentenceEncoderLayer, self).__init__() - self.embedding_dim = sent_enc_layer.embedding_dim - self.dropout = sent_enc_layer.dropout - self.activation_dropout = sent_enc_layer.activation_dropout - - # Initialize blocks - self.activation_fn = sent_enc_layer.activation_fn - self.self_attn = sent_enc_layer.self_attn - - self.dropout1 = sent_enc_layer.dropout1 - self.dropout2 = sent_enc_layer.dropout2 - self.dropout3 = sent_enc_layer.dropout3 - - self.layer_norm_first = sent_enc_layer.layer_norm_first - - # layer norm associated with the self attention layer - self.self_attn_layer_norm = sent_enc_layer.self_attn_layer_norm - self.fc1 = sent_enc_layer.fc1 - self.fc2 = sent_enc_layer.fc2 - - # layer norm associated with the position wise feed-forward NN - self.final_layer_norm = sent_enc_layer.final_layer_norm - - def forward( - self, - x, - self_attn_mask=None, - self_attn_padding_mask=None, - need_weights=None, - att_args=None, - ): - x, attn = super().forward( - x, self_attn_mask, self_attn_padding_mask, need_weights, att_args - ) - return x - - -# TODO retire SharedEncoder -class SharedEncoder(FairseqEncoder): - def __init__(self, wav2vec_enc, mbart_enc, adaptor, shared_layers): - super().__init__(None) - self.w2v_encoder = wav2vec_enc - self.shared_layers = self.w2v_encoder.w2v_model.encoder.layers[-shared_layers:] - self.w2v_encoder.w2v_model.encoder.layers = ( - self.w2v_encoder.w2v_model.encoder.layers[:-shared_layers] - ) - self.adaptor = adaptor - if self.shared_layers[-1].layer_norm_first: - self.final_layer_norm = mbart_enc.layer_norm - else: - mbart_enc.layer_norm = None - self.final_layer_norm = None - shared_layer_from = len(mbart_enc.layers) - shared_layers - if shared_layer_from < 0: - shared_layer_from = 0 - for layer_id, layer in enumerate(self.shared_layers): - mbart_enc.layers[ - shared_layer_from + layer_id - ] = TransformerSentenceEncoderLayerStd(layer) - - def forward(self, src_tokens, src_lengths=None, **kwargs): - padding_mask = lengths_to_padding_mask(src_lengths) - if not padding_mask.any(): - padding_mask = None - - out = self.w2v_encoder.forward(src_tokens, padding_mask, tbc=True) - x = out["encoder_out"] - enc_padding_mask = None - if out["encoder_padding_mask"] is not None: - enc_padding_mask = out["encoder_padding_mask"].transpose( - 0, 1 - ) # T X B --> B X T - - x, enc_padding_mask = self.adaptor(x, enc_padding_mask) - for layer in self.shared_layers: - x, _ = layer(x, enc_padding_mask) - if self.final_layer_norm is not None: - x = self.final_layer_norm(x) - - return { - "encoder_out": [x], # T x B x C - "encoder_padding_mask": [enc_padding_mask] - if enc_padding_mask is not None - else [], # B x T - "encoder_embedding": [], # B x T x C - "encoder_states": [], # List[T x B x C] - "src_tokens": [], - "src_lengths": [], - } - - -class StackedWav2VecEncoderWithAdaptor(FairseqEncoder): - def __init__( - self, - wav2vec_enc, - mbart_enc_layers, - mbart_layer_norm, - adaptor, - drop_w2v_layers=0, - ): - super().__init__(None) - self.w2v_encoder = wav2vec_enc - self.adaptor = adaptor - self.mbart_encoder_layers = mbart_enc_layers - self.final_layer_norm = mbart_layer_norm - if drop_w2v_layers > 0: - self.w2v_encoder.w2v_model.encoder.layers = ( - self.w2v_encoder.w2v_model.encoder.layers[:-drop_w2v_layers] - ) - - def forward(self, src_tokens, src_lengths=None, return_all_hiddens=False, **kwargs): - padding_mask = lengths_to_padding_mask(src_lengths) - if not padding_mask.any(): - padding_mask = None - - out = self.w2v_encoder.forward(src_tokens, padding_mask, tbc=True) - x = out["encoder_out"] - enc_padding_mask = None - if out["padding_mask"] is not None: - enc_padding_mask = out["padding_mask"] # B X T - - x, enc_padding_mask = self.adaptor(x, enc_padding_mask) - encoder_states = [] - for layer in self.mbart_encoder_layers: - x = layer(x, enc_padding_mask) - if return_all_hiddens: - encoder_states.append(x) - if self.final_layer_norm is not None: - x = self.final_layer_norm(x) - - return { - "encoder_out": [x], # T x B x C - "encoder_padding_mask": [enc_padding_mask] - if enc_padding_mask is not None - else [], # B x T - "encoder_embedding": [], # B x T x C - "encoder_states": encoder_states, # List[T x B x C] - "src_tokens": [], - "src_lengths": [], - } - - def reorder_encoder_out(self, encoder_out, new_order): - new_encoder_out = ( - [] - if len(encoder_out["encoder_out"]) == 0 - else [x.index_select(1, new_order) for x in encoder_out["encoder_out"]] - ) - - new_encoder_padding_mask = ( - [] - if len(encoder_out["encoder_padding_mask"]) == 0 - else [ - x.index_select(0, new_order) - for x in encoder_out["encoder_padding_mask"] - ] - ) - - new_encoder_embedding = ( - [] - if len(encoder_out["encoder_embedding"]) == 0 - else [ - x.index_select(0, new_order) for x in encoder_out["encoder_embedding"] - ] - ) - - encoder_states = encoder_out["encoder_states"] - if len(encoder_states) > 0: - for idx, state in enumerate(encoder_states): - encoder_states[idx] = state.index_select(1, new_order) - - return { - "encoder_out": new_encoder_out, # T x B x C - "encoder_padding_mask": new_encoder_padding_mask, # B x T - "encoder_embedding": new_encoder_embedding, # B x T x C - "encoder_states": encoder_states, # List[T x B x C] - "src_tokens": [], # B x T - "src_lengths": [], # B x 1 - } - - -# Note: -# dual input transformer: -# encoder: wav2vec for speech + mbart encoder for text -# decoder: mbart decoder for text -@register_model("dual_input_xm_transformer") -class DualInputXMTransformerModel(DualInputS2TTransformerModel): - def __init__(self, encoder, decoder): - super().__init__(encoder, decoder) - - @staticmethod - def add_args(parser): - """Add model-specific arguments to the parser.""" - # wav2vec encoder - Wav2VecEncoderWithAdaptor.add_args(parser) - # add_decoder_args(parser) - # mbart Transformer - parser.add_argument( - "--activation-fn", - type=str, - default="relu", - choices=utils.get_available_activation_fns(), - help="activation function to use", - ) - - parser.add_argument( - "--mbart-dropout", type=float, metavar="D", help="dropout probability" - ) - parser.add_argument( - "--mbart-attention-dropout", - type=float, - metavar="D", - help="dropout probability for attention weights", - ) - parser.add_argument( - "--mbart-activation-dropout", - type=float, - metavar="D", - help="dropout probability after activation in FFN.", - ) - - parser.add_argument( - "--encoder-embed-dim", - type=int, - metavar="N", - help="encoder embedding dimension", - ) - parser.add_argument( - "--encoder-ffn-embed-dim", - type=int, - metavar="N", - help="encoder embedding dimension for FFN", - ) - parser.add_argument( - "--encoder-layers", type=int, metavar="N", help="num encoder layers" - ) - parser.add_argument( - "--encoder-attention-heads", - type=int, - metavar="N", - help="num encoder attention heads", - ) - parser.add_argument( - "--encoder-normalize-before", - action="store_true", - help="apply layernorm before each encoder block", - ) - - parser.add_argument( - "--decoder-embed-dim", - type=int, - metavar="N", - help="decoder embedding dimension", - ) - parser.add_argument( - "--decoder-ffn-embed-dim", - type=int, - metavar="N", - help="decoder embedding dimension for FFN", - ) - parser.add_argument( - "--decoder-layers", type=int, metavar="N", help="num decoder layers" - ) - parser.add_argument( - "--decoder-attention-heads", - type=int, - metavar="N", - help="num decoder attention heads", - ) - parser.add_argument( - "--decoder-normalize-before", - action="store_true", - help="apply layernorm before each decoder block", - ) - parser.add_argument( - "--layernorm-embedding", - action="store_true", - help="add layernorm to embedding", - ) - parser.add_argument( - "--no-scale-embedding", - action="store_true", - help="if True, dont scale embeddings", - ) - parser.add_argument( - "--load-pretrained-mbart-from", - type=str, - metavar="STR", - help="model to take text encoder decoder weights from (for initialization)", - ) - # parser.add_argument("--finetune-w2v-params", type=str, metavar="STR", - # help="comma-separated param strings to finetune.") - parser.add_argument( - "--finetune-mbart-decoder-params", - type=str, - metavar="STR", - help="comma-separated param strings to finetune.", - ) - parser.add_argument( - "--finetune-mbart-encoder-params", - type=str, - metavar="STR", - help="comma-separated param strings to finetune.", - ) - parser.add_argument( - "--skip-encoder-projection", - action="store_true", - help="skip the projection layer in encoder", - ) - - parser.add_argument( - "--enc-grad-mult", - type=float, - metavar="V", - default=1.0, - help="multiply enc1 and enc2 gradient by V", - ) - parser.add_argument( - "--enc2-along-grad-mult", - type=float, - metavar="V", - default=1.0, - help="multiply enc2 gradient by V if only enc2 is used", - ) - parser.add_argument( - "--text-input-cost-ratio", - type=float, - default=1.0, - metavar="V", - help="text input cost ratio relative to speech input cost", - ) - parser.add_argument( - "--stack-w2v-mbart-encoder", - action="store_true", - help="stack w2v and mbart encoder", - ) - parser.add_argument( - "--stack-w2v-mbart-nonorm-encoder", - action="store_true", - help="stack w2v and mbart encoder", - ) - parser.add_argument( - "--no-final-norm-decoder", action="store_true", help="no layer norm" - ) - parser.add_argument( - "--drop-w2v-layers", - type=int, - default=0, - metavar="N", - help="drop w2v encoder layers", - ) - - parser.add_argument( - "--share-w2v-text-encoder", - action="store_true", - help="share w2v encoder layers with text encoder", - ) - parser.add_argument( - "--shared-w2v-layers", - type=int, - default=0, - metavar="N", - help="shared encoder layers from w2v encoder", - ) - - @classmethod - def build_encoder(cls, args, task): - _args = copy.deepcopy(args) - _args.dropout = args.mbart_dropout - _args.attention_dropout = args.mbart_attention_dropout - _args.activation_dropout = args.mbart_activation_dropout - _args.max_source_positions = 1024 - enc_emb = nn.Embedding( - len(task.src_dict), _args.encoder_embed_dim, task.src_dict.pad() - ) - text_encoder = TransformerEncoder(_args, task.src_dict, enc_emb) - spch_encoder = Wav2VecEncoderWithAdaptor(args) - if getattr(args, "load_pretrained_mbart_from", None): - text_encoder = checkpoint_utils.load_pretrained_component_from_model( - component=text_encoder, checkpoint=args.load_pretrained_mbart_from - ) - if getattr(args, "stack_w2v_mbart_encoder", False): - assert getattr(args, "share_w2v_text_encoder", False) is False - spch_encoder = StackedWav2VecEncoderWithAdaptor( - spch_encoder.w2v_encoder, - text_encoder.layers, - text_encoder.layer_norm, - spch_encoder.adaptor, - args.drop_w2v_layers, - ) - elif getattr(args, "stack_w2v_mbart_nonorm_encoder", False): - text_encoder.layer_norm = None - spch_encoder = StackedWav2VecEncoderWithAdaptor( - spch_encoder.w2v_encoder, - text_encoder.layers, - text_encoder.layer_norm, - spch_encoder.adaptor, - args.drop_w2v_layers, - ) - elif getattr(args, "share_w2v_text_encoder", False): - spch_encoder = SharedEncoder( - spch_encoder.w2v_encoder, - text_encoder, - spch_encoder.adaptor, - args.shared_w2v_layers, - ) - - for k, p in spch_encoder.named_parameters(): - # Freeze pretrained models by default - if safe_hasattr( - args, "finetune_w2v_params" - ) and need_finetuning(args.finetune_w2v_params, k): - p.requires_grad = True - else: - p.requires_grad = False - for k, p in text_encoder.named_parameters(): - # Freeze pretrained models by default - if safe_hasattr( - args, "finetune_mbart_encoder_params" - ) and need_finetuning( - args.finetune_mbart_encoder_params, k - ): - p.requires_grad = True - else: - p.requires_grad = False - cross_attentive_loss_before_last_layer = ( - 0 if getattr(args, "attentive_cost_regularization", 0.0) > 0.0 else -1 - ) - encoder = DualInputEncoder( - args, - spch_encoder, - text_encoder, - task.src_dict, - cross_attentive_loss_before_last_layer, - ) - return encoder - - @classmethod - def build_decoder(cls, args, task): - _args = copy.deepcopy(args) - _args.dropout = args.mbart_dropout - _args.attention_dropout = args.mbart_attention_dropout - _args.activation_dropout = args.mbart_activation_dropout - _args.max_target_positions = 1024 - dec_emb = nn.Embedding( - len(task.tgt_dict), _args.encoder_embed_dim, task.tgt_dict.pad() - ) - decoder = TransformerDecoder(_args, task.tgt_dict, dec_emb) - if getattr(args, "load_pretrained_mbart_from", None): - decoder = checkpoint_utils.load_pretrained_component_from_model( - component=decoder, checkpoint=args.load_pretrained_mbart_from - ) - if getattr(args, "no_final_norm_decoder", False): - decoder.layer_norm = None - for k, p in decoder.named_parameters(): - # Freeze pretrained models by default - if safe_hasattr( - args, "finetune_mbart_decoder_params" - ) and need_finetuning( - args.finetune_mbart_decoder_params, k - ): - p.requires_grad = True - else: - p.requires_grad = False - - compute_cross_attentive_loss = ( - True if getattr(args, "attentive_cost_regularization", 0.0) > 0.0 else False - ) - cross_attentive_loss_without_norm = getattr( - args, "attentive_cost_without_normalize", False - ) - cross_attentive_loss_reverse = ( - False # getattr(args, "attentive_cost_reverse", False) - ) - decoder = TransformerMultiInputDecoder( - dictionary=task.target_dictionary, - spch_decoder=decoder, - text_decoder=decoder, - compute_cross_attentive_loss=compute_cross_attentive_loss, - cross_attentive_loss_with_norm=True - if not cross_attentive_loss_without_norm - else False, - cross_attentive_loss_reverse=cross_attentive_loss_reverse, - ) - return decoder - - @classmethod - def build_model(cls, args, task): - """Build a new model instance.""" - # make sure that all args are properly defaulted - # (in case there are any new ones) - dualinputxmtransformer_base(args) - - encoder = cls.build_encoder(args, task) - decoder = cls.build_decoder(args, task) - return cls(encoder, decoder) - - -@register_model_architecture("dual_input_xm_transformer", "dualinputxmtransformer_base") -def dualinputxmtransformer_base(args): - # wav2vec encoder - set_default_w2v_encoder_args(args) - set_default_adaptor_args(args) - - # mbart model - args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 1024) - args.encoder_ffn_embed_dim = getattr( - args, "encoder_ffn_embed_dim", 4 * args.encoder_embed_dim - ) - args.encoder_layers = getattr(args, "encoder_layers", 12) - args.encoder_attention_heads = getattr(args, "encoder_attention_heads", 16) - args.encoder_normalize_before = getattr(args, "encoder_normalize_before", True) - args.encoder_layerdrop = getattr(args, "encoder_layerdrop", 0) - args.encoder_learned_pos = getattr(args, "encoder_learned_pos", True) - - args.decoder_embed_path = getattr(args, "decoder_embed_path", None) - args.decoder_embed_dim = getattr(args, "decoder_embed_dim", 1024) - args.decoder_ffn_embed_dim = getattr(args, "decoder_ffn_embed_dim", 4 * 1024) - args.decoder_layers = getattr(args, "decoder_layers", 12) - args.decoder_attention_heads = getattr(args, "decoder_attention_heads", 16) - args.decoder_normalize_before = getattr(args, "decoder_normalize_before", True) - args.decoder_learned_pos = getattr(args, "decoder_learned_pos", True) - args.decoder_layerdrop = getattr(args, "decoder_layerdrop", 0.0) - - args.adaptive_input = getattr(args, "adaptive_input", False) - - args.mbart_attention_dropout = getattr(args, "mbart_attention_dropout", 0.0) - args.mbart_activation_dropout = getattr(args, "mbart_activation_dropout", 0.0) - args.mbart_dropout = getattr(args, "mbart_dropout", 0.1) - args.adaptive_softmax_cutoff = getattr(args, "adaptive_softmax_cutoff", None) - args.adaptive_softmax_dropout = getattr(args, "adaptive_softmax_dropout", 0) - args.share_decoder_input_output_embed = getattr( - args, "share_decoder_input_output_embed", True - ) - args.no_token_positional_embeddings = getattr( - args, "no_token_positional_embeddings", False - ) - - args.decoder_output_dim = getattr( - args, "decoder_output_dim", args.decoder_embed_dim - ) - args.decoder_input_dim = getattr(args, "decoder_input_dim", args.decoder_embed_dim) - - args.no_scale_embedding = getattr(args, "no_scale_embedding", False) - args.quant_noise_pq = getattr(args, "quant_noise_pq", 0) - args.layernorm_embedding = getattr(args, "layernorm_embedding", True) - - args.activation_fn = getattr(args, "activation_fn", "gelu") - args.pooler_activation_fn = getattr(args, "pooler_activation_fn", "tanh") - args.pooler_dropout = getattr(args, "pooler_dropout", 0.0) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/scripts/convert_model.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/scripts/convert_model.py deleted file mode 100644 index 47ae9352557a509a38c03ff87b53b54dac6a4ed5..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/scripts/convert_model.py +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import re -from collections import OrderedDict - -import torch -from fairseq.file_io import PathManager - - -def is_update(param_name, module_name): - if module_name in param_name: - return True - return False - - -def load_checkpoint(src_cpt): - - with PathManager.open(src_cpt, "rb") as f: - state_src = torch.load( - f, - map_location=( - lambda s, _: torch.serialization.default_restore_location(s, "cpu") - ), - ) - - return state_src - - -def save_checkpoint(tgt_cpt, states): - - with PathManager.open(tgt_cpt, "wb") as f: - torch.save( - states, - f, - ) - - -# convert the pre-trained model into bart model -def main(): - parser = argparse.ArgumentParser() - # fmt: off - parser.add_argument('--input-model', required=True, - help='Input checkpoint file path.') - parser.add_argument('--output-model', required=True, - help='output checkpoint file path.') - # fmt: on - args = parser.parse_args() - print(args) - - states = load_checkpoint(args.input_model) - model = states["model"] - new_model = OrderedDict() - for key in model.keys(): - if re.search("^encoder.text_encoder", key): - new_key = re.sub("encoder.text_encoder", "encoder", key) - new_model[new_key] = model[key] - elif re.search("^decoder.text_decoder", key): - new_key = re.sub("decoder.text_decoder", "decoder", key) - new_model[new_key] = model[key] - states["model"] = new_model - save_checkpoint(args.output_model, states) - - -if __name__ == "__main__": - main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/scripts/g2p_encode.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/scripts/g2p_encode.py deleted file mode 100644 index 9db779396f492e3f71b08d7b895beb81d8e46bc9..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/scripts/g2p_encode.py +++ /dev/null @@ -1,191 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import itertools -import logging -import re -import time - -from g2p_en import G2p - -logger = logging.getLogger(__name__) - -FAIL_SENT = "FAILED_SENTENCE" - - -def parse(): - parser = argparse.ArgumentParser() - parser.add_argument("--data-path", type=str, required=True) - parser.add_argument("--out-path", type=str, required=True) - parser.add_argument("--lower-case", action="store_true") - parser.add_argument("--do-filter", action="store_true") - parser.add_argument("--use-word-start", action="store_true") - parser.add_argument("--dup-vowel", default=1, type=int) - parser.add_argument("--dup-consonant", default=1, type=int) - parser.add_argument("--no-punc", action="store_true") - parser.add_argument("--reserve-word", type=str, default="") - parser.add_argument( - "--reserve-first-column", - action="store_true", - help="first column is sentence id", - ) - ### - parser.add_argument("--parallel-process-num", default=1, type=int) - parser.add_argument("--logdir", default="") - args = parser.parse_args() - return args - - -def process_sent(sent, g2p, res_wrds, args): - sents = pre_process_sent(sent, args.do_filter, args.lower_case, res_wrds) - pho_seqs = [do_g2p(g2p, s, res_wrds, i == 0) for i, s in enumerate(sents)] - pho_seq = ( - [FAIL_SENT] - if [FAIL_SENT] in pho_seqs - else list(itertools.chain.from_iterable(pho_seqs)) - ) - if args.no_punc: - pho_seq = remove_punc(pho_seq) - if args.dup_vowel > 1 or args.dup_consonant > 1: - pho_seq = dup_pho(pho_seq, args.dup_vowel, args.dup_consonant) - if args.use_word_start: - pho_seq = add_word_start(pho_seq) - return " ".join(pho_seq) - - -def remove_punc(sent): - ns = [] - regex = re.compile("[^a-zA-Z0-9 ]") - for p in sent: - if (not regex.search(p)) or p == FAIL_SENT: - if p == " " and (len(ns) == 0 or ns[-1] == " "): - continue - ns.append(p) - return ns - - -def do_g2p(g2p, sent, res_wrds, is_first_sent): - if sent in res_wrds: - pho_seq = [res_wrds[sent]] - else: - pho_seq = g2p(sent) - if not is_first_sent: - pho_seq = [" "] + pho_seq # add space to separate - return pho_seq - - -def pre_process_sent(sent, do_filter, lower_case, res_wrds): - if do_filter: - sent = re.sub("-", " ", sent) - sent = re.sub("—", " ", sent) - if len(res_wrds) > 0: - wrds = sent.split() - wrds = ["SPLIT_ME " + w + " SPLIT_ME" if w in res_wrds else w for w in wrds] - sents = [x.strip() for x in " ".join(wrds).split("SPLIT_ME") if x.strip() != ""] - else: - sents = [sent] - if lower_case: - sents = [s.lower() if s not in res_wrds else s for s in sents] - return sents - - -def dup_pho(sent, dup_v_num, dup_c_num): - """ - duplicate phoneme defined as cmudict - http://www.speech.cs.cmu.edu/cgi-bin/cmudict - """ - if dup_v_num == 1 and dup_c_num == 1: - return sent - ns = [] - for p in sent: - ns.append(p) - if re.search(r"\d$", p): - for i in range(1, dup_v_num): - ns.append(f"{p}-{i}P") - elif re.search(r"\w", p): - for i in range(1, dup_c_num): - ns.append(f"{p}-{i}P") - return ns - - -def add_word_start(sent): - ns = [] - do_add = True - ws = "▁" - for p in sent: - if do_add: - p = ws + p - do_add = False - if p == " ": - do_add = True - else: - ns.append(p) - return ns - - -def load_reserve_word(reserve_word): - if reserve_word == "": - return [] - with open(reserve_word, "r") as fp: - res_wrds = [x.strip().split() for x in fp.readlines() if x.strip() != ""] - assert sum([0 if len(x) == 2 else 1 for x in res_wrds]) == 0 - res_wrds = dict(res_wrds) - return res_wrds - - -def process_sents(sents, args): - g2p = G2p() - out_sents = [] - res_wrds = load_reserve_word(args.reserve_word) - for sent in sents: - col1 = "" - if args.reserve_first_column: - col1, sent = sent.split(None, 1) - sent = process_sent(sent, g2p, res_wrds, args) - if args.reserve_first_column and col1 != "": - sent = f"{col1} {sent}" - out_sents.append(sent) - return out_sents - - -def main(): - args = parse() - out_sents = [] - with open(args.data_path, "r") as fp: - sent_list = [x.strip() for x in fp.readlines()] - if args.parallel_process_num > 1: - try: - import submitit - except ImportError: - logger.warn( - "submitit is not found and only one job is used to process the data" - ) - submitit = None - - if args.parallel_process_num == 1 or submitit is None: - out_sents = process_sents(sent_list, args) - else: - # process sentences with parallel computation - lsize = len(sent_list) // args.parallel_process_num + 1 - executor = submitit.AutoExecutor(folder=args.logdir) - executor.update_parameters(timeout_min=1000, cpus_per_task=4) - jobs = [] - for i in range(args.parallel_process_num): - job = executor.submit( - process_sents, sent_list[lsize * i : lsize * (i + 1)], args - ) - jobs.append(job) - is_running = True - while is_running: - time.sleep(5) - is_running = sum([job.done() for job in jobs]) < len(jobs) - out_sents = list(itertools.chain.from_iterable([job.result() for job in jobs])) - with open(args.out_path, "w") as fp: - fp.write("\n".join(out_sents) + "\n") - - -if __name__ == "__main__": - main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/tasks/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/tasks/__init__.py deleted file mode 100644 index 5fc5d9e21b2b5d272bd67610958efbdb1eecb158..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/tasks/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import importlib -import os - diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/tasks/pair_denoising.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/tasks/pair_denoising.py deleted file mode 100644 index ab4e217f12b7fa46571f0ee87115e80a8267bac1..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/tasks/pair_denoising.py +++ /dev/null @@ -1,446 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import itertools -import logging -import os -import re - -import numpy as np -import torch -from examples.speech_text_joint_to_text.data.pair_denoising_dataset import ( - LanguagePairDenoisingDataset, -) -from fairseq import utils -from fairseq.data import ( - ConcatDataset, - Dictionary, - LanguagePairDataset, - ResamplingDataset, - TransformEosConcatLangPairDataset, - TransformEosLangPairDataset, - data_utils, - indexed_dataset, -) -from fairseq.data.encoders.utils import get_whole_word_mask -from fairseq.tasks import register_task -from fairseq.tasks.translation import TranslationTask - -logger = logging.getLogger(__name__) - - -def gen_whole_word_mask(args, dictionary): - def is_beginning_of_word(i): - if i < dictionary.nspecial: - # special elements are always considered beginnings - return True - tok = dictionary[i] - if tok.startswith("madeupword"): - return True - - if tok in ["", "", "", ""]: - return True - return tok.startswith("\u2581") - - if args.use_mask_whole_words: - mask_whole_words = torch.ByteTensor( - list(map(is_beginning_of_word, range(len(dictionary)))) - ) - else: - # it will mask every token as word leading token, since no bpe model is loaded for phoneme tokens - return get_whole_word_mask(args, dictionary) - return mask_whole_words - - -@register_task("paired_denoising") -class PairedDenoisingTask(TranslationTask): - - LANG_TAG_TEMPLATE = "" # Tag for language (target) - - @staticmethod - def add_args(parser): - TranslationTask.add_args(parser) - # bart setting - parser.add_argument( - "--mask", - default=0.0, - type=float, - help="fraction of words/subwords that will be masked", - ) - parser.add_argument( - "--mask-random", - default=0.0, - type=float, - help="instead of using [MASK], use random token this often", - ) - parser.add_argument( - "--insert", - default=0.0, - type=float, - help="insert this percentage of additional random tokens", - ) - parser.add_argument( - "--poisson-lambda", - default=3.0, - type=float, - help="randomly shuffle sentences for this proportion of inputs", - ) - parser.add_argument( - "--mask-length", - default="span-poisson", - type=str, - choices=["subword", "word", "span-poisson"], - help="mask length to choose", - ) - parser.add_argument( - "--replace-length", - default=1, - type=int, - help="when masking N tokens, replace with 0, 1, or N tokens (use -1 for N)", - ) - - # multi-lingual - parser.add_argument( - "--multilang-sampling-alpha", - type=float, - default=1.0, - help="smoothing alpha for sample ratios across multiple datasets", - ) - parser.add_argument( - "--lang-pairs", - default="", - metavar="PAIRS", - help="comma-separated list of language pairs (in training order): phnen-en,phnfr-fr,phnit-it. Do masking", - ) - parser.add_argument( - "--lang-pairs-bitext", - default="", - metavar="PAIRS", - help="comma-separated list of language pairs (in training order): en-de,en-fr,de-fr. No masking", - ) - parser.add_argument("--add-src-lang-token", default=False, action="store_true") - parser.add_argument("--add-tgt-lang-token", default=False, action="store_true") - parser.add_argument( - "--no-whole-word-mask-langs", - type=str, - default="", - metavar="N", - help="languages without spacing between words dont support whole word masking", - ) - parser.add_argument( - "--use-mask-whole-words", default=False, action="store_true" - ) - - @classmethod - def setup_task(cls, args, **kwargs): - """Setup the task.""" - paths = args.data.split(":") - assert len(paths) > 0 - src_dict = Dictionary.load( - os.path.join(paths[0], "src_dict.txt") - ) # assume all languages share a source dictionary - tgt_dict = Dictionary.load( - os.path.join(paths[0], "tgt_dict.txt") - ) # assume all languages share a target dictionary - - lang_pairs = args.lang_pairs + "," + args.lang_pairs_bitext - lang_pairs = re.sub(",$", "", re.sub("^,", "", lang_pairs)) - src_langs = [lp.split("-")[0] for lp in lang_pairs.split(",")] - tgt_langs = [lp.split("-")[1] for lp in lang_pairs.split(",")] - - if args.add_src_lang_token: - for lang in src_langs: - assert ( - src_dict.index(PairedDenoisingTask.LANG_TAG_TEMPLATE.format(lang)) - != src_dict.unk() - ) - if args.add_tgt_lang_token: - for lang in tgt_langs: - assert ( - tgt_dict.index(PairedDenoisingTask.LANG_TAG_TEMPLATE.format(lang)) - != tgt_dict.unk() - ) - - logger.info("source dictionary: {} types".format(len(src_dict))) - logger.info("target dictionary: {} types".format(len(tgt_dict))) - if not hasattr(args, "shuffle_instance"): - args.shuffle_instance = False - return cls(args, src_dict, tgt_dict) - - def __init__(self, args, src_dict, tgt_dict): - super().__init__(args, src_dict, tgt_dict) - # check mask token - self.mask_idx = self.src_dict.index("") - assert self.mask_idx != self.src_dict.unk() - self.lang_pairs = args.lang_pairs - self.lang_pairs_bitext = args.lang_pairs_bitext - self.args = args - - @classmethod - def language_pair_denoising_dataset( - cls, - data_path, - do_mask, - split, - src, - src_dict, - tgt, - tgt_dict, - mask_idx, - mask_whole_words, - seed, - args, - dataset_impl, - combine=False, - left_pad_source=True, - left_pad_target=False, - max_source_positions=1024, - max_target_positions=1024, - shuffle=True, - src_lang_id=None, - tgt_lang_id=None, - ): - def split_exists(split, src, tgt, lang, data_path): - filename = os.path.join( - data_path, "{}.{}-{}.{}".format(split, src, tgt, lang) - ) - return indexed_dataset.dataset_exists(filename, impl=dataset_impl) - - src_datasets = [] - tgt_datasets = [] - - for k in itertools.count(): - split_k = split + (str(k) if k > 0 else "") - - # infer langcode - if split_exists(split_k, src, tgt, src, data_path): - prefix = os.path.join(data_path, "{}.{}-{}.".format(split_k, src, tgt)) - elif split_exists(split_k, tgt, src, src, data_path): - prefix = os.path.join(data_path, "{}.{}-{}.".format(split_k, tgt, src)) - else: - if k > 0: - break - else: - raise FileNotFoundError( - "Dataset not found: {} ({})".format(split, data_path) - ) - - src_dataset = data_utils.load_indexed_dataset( - prefix + src, src_dict, dataset_impl - ) - src_datasets.append(src_dataset) - - tgt_dataset = data_utils.load_indexed_dataset( - prefix + tgt, tgt_dict, dataset_impl - ) - if tgt_dataset is not None: - tgt_datasets.append(tgt_dataset) - - logger.info( - "{} {} {}-{} {} examples".format( - data_path, split_k, src, tgt, len(src_datasets[-1]) - ) - ) - - if not combine: - break - - assert len(src_datasets) == len(tgt_datasets) or len(tgt_datasets) == 0 - - if len(src_datasets) == 1: - src_dataset = src_datasets[0] - tgt_dataset = tgt_datasets[0] if len(tgt_datasets) > 0 else None - else: - sample_ratios = [1] * len(src_datasets) - src_dataset = ConcatDataset(src_datasets, sample_ratios) - if len(tgt_datasets) > 0: - tgt_dataset = ConcatDataset(tgt_datasets, sample_ratios) - else: - tgt_dataset = None - - eos = None - - tgt_dataset_sizes = tgt_dataset.sizes if tgt_dataset is not None else None - if not do_mask: - return LanguagePairDataset( - src_dataset, - src_dataset.sizes, - src_dict, - tgt_dataset, - tgt_dataset_sizes, - tgt_dict, - left_pad_source=left_pad_source, - left_pad_target=left_pad_target, - eos=eos, - shuffle=shuffle, - src_lang_id=src_lang_id, - tgt_lang_id=tgt_lang_id, - ) - - return LanguagePairDenoisingDataset( - src_dataset, - src_dataset.sizes, - src_dict, - tgt_dataset, - tgt_dataset_sizes, - tgt_dict, - mask_idx, - mask_whole_words, - seed, - args, - left_pad_source=left_pad_source, - left_pad_target=left_pad_target, - eos=eos, - shuffle=shuffle, - src_lang_id=src_lang_id, - tgt_lang_id=tgt_lang_id, - ) - - def _get_sample_prob(self, dataset_lens): - """ - Get smoothed sampling porbability by languages. This helps low resource - languages by upsampling them. - """ - prob = dataset_lens / dataset_lens.sum() - smoothed_prob = prob ** self.args.multilang_sampling_alpha - smoothed_prob = smoothed_prob / smoothed_prob.sum() - return smoothed_prob - - def resample_datasets(self, lang_datasets, lang_pairs_all, epoch): - # For train subset, additionally up or down sample languages. - if self.args.multilang_sampling_alpha == 1.0: - return lang_datasets - - dataset_lengths = np.array( - [len(d) for d in lang_datasets], - dtype=float, - ) - sample_probs = self._get_sample_prob(dataset_lengths) - logger.info( - "Sample probability by language pair: {}".format( - { - lp: "{0:.4f}".format(sample_probs[id]) - for id, lp in enumerate(lang_pairs_all) - } - ) - ) - size_ratio = (sample_probs * dataset_lengths.sum()) / dataset_lengths - logger.info( - "Up/Down Sampling ratio by language: {}".format( - { - lp: "{0:.2f}".format(size_ratio[id]) - for id, lp in enumerate(lang_pairs_all) - } - ) - ) - - resampled_lang_datasets = [ - ResamplingDataset( - lang_datasets[i], - size_ratio=size_ratio[i], - seed=self.args.seed, - epoch=epoch, - replace=size_ratio[i] >= 1.0, - ) - for i, d in enumerate(lang_datasets) - ] - return resampled_lang_datasets - - def load_dataset_only( - self, split, lang_pairs, do_mask=True, epoch=1, combine=False - ): - paths = utils.split_paths(self.args.data) - assert len(paths) > 0 - data_path = paths[(epoch - 1) % len(paths)] - - # TODO unk token will be considered as first word too, though it might be an unknown phoneme within a word - # get_whole_word_mask returns a tensor (size V by 1 ) to indicate if a token is a word start token - mask_whole_src_words = gen_whole_word_mask(self.args, self.src_dict) - language_without_segmentations = self.args.no_whole_word_mask_langs.split(",") - lang_datasets = [] - eos_bos = [] - lang_pairs = lang_pairs.split(",") if lang_pairs != "" else [] - assert len(lang_pairs) > 0 - for lp in lang_pairs: - src, tgt = lp.split("-") - lang_mask_whole_src_words = ( - mask_whole_src_words - if src not in language_without_segmentations - else None - ) - - end_token = ( - self.source_dictionary.index( - PairedDenoisingTask.LANG_TAG_TEMPLATE.format(src) - ) - if self.args.add_src_lang_token - else None - ) - bos_token = ( - self.target_dictionary.index( - PairedDenoisingTask.LANG_TAG_TEMPLATE.format(tgt) - ) - if self.args.add_tgt_lang_token - else None - ) - src_lang_id = None - - if self.args.add_src_lang_token or self.args.add_tgt_lang_token: - eos_bos.append((end_token, bos_token)) - - dataset = PairedDenoisingTask.language_pair_denoising_dataset( - data_path, - do_mask, - split, - src, - self.source_dictionary, - tgt, - self.target_dictionary, - self.mask_idx, - lang_mask_whole_src_words, - self.args.seed, - self.args, - self.args.dataset_impl, - combine=combine, - left_pad_source=utils.eval_bool(self.args.left_pad_source), - left_pad_target=utils.eval_bool(self.args.left_pad_target), - max_source_positions=self.args.max_source_positions, - max_target_positions=self.args.max_target_positions, - src_lang_id=src_lang_id, - ) - - lang_datasets.append(dataset) - - if len(lang_datasets) == 0: - return - elif len(lang_datasets) == 1: - dataset = lang_datasets[0] - if self.args.add_src_lang_token or self.args.add_tgt_lang_token: - end_token, bos_token = eos_bos[0] - dataset = TransformEosLangPairDataset( - dataset, - src_eos=self.source_dictionary.eos(), - new_src_eos=end_token, - tgt_bos=self.target_dictionary.eos(), - new_tgt_bos=bos_token, - ) - else: - end_tokens = [item[0] for item in eos_bos if item[0] is not None] - bos_tokens = [item[1] for item in eos_bos if item[1] is not None] - lang_datasets = self.resample_datasets(lang_datasets, lang_pairs, epoch) - dataset = TransformEosConcatLangPairDataset( - lang_datasets, - self.source_dictionary.eos(), - self.target_dictionary.eos(), - new_src_eos=end_tokens, - new_tgt_bos=bos_tokens, - ) - return dataset - - # split in (train, valid, test, ...) - def load_dataset(self, split, epoch=1, combine=False, **kwargs): - self.datasets[split] = self.load_dataset_only( - split, self.lang_pairs, epoch=epoch, combine=combine - ) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/tasks/speech_text_denoise_pretrain.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/tasks/speech_text_denoise_pretrain.py deleted file mode 100644 index f592633c08ef0b5f4b17bd388d2fab2612827eca..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/tasks/speech_text_denoise_pretrain.py +++ /dev/null @@ -1,654 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. -import logging -import os -import re -from argparse import Namespace -from pathlib import Path - -from fairseq.data import ConcatDataset, Dictionary, encoders -from fairseq.data.audio.multi_modality_dataset import ( - FileAudioDatasetWrapper, - ModalityDatasetItem, - MultiModalityDataset, -) -from fairseq.data.audio.speech_to_text_joint_dataset import ( - S2TJointDataConfig, - SpeechToTextJointDatasetCreator, -) -from fairseq.data.iterators import GroupedEpochBatchIterator -from fairseq.tasks import register_task - -from .pair_denoising import PairedDenoisingTask - -logger = logging.getLogger(__name__) - - -@register_task("speech_text_joint_denoising") -class SpeechTextJointDenoisingPreTask(PairedDenoisingTask): - """ - Joint denoising training task for speech and text. - """ - - SIL_TOKEN = "sil" - - @classmethod - def add_args(cls, parser): - PairedDenoisingTask.add_args(parser) - # set max tokens and position - parser.add_argument( - "--max-text-tokens", - type=int, - metavar="N", - default=1024, - help="maximum samples for encoder text input ", - ) - parser.add_argument( - "--max-speech-tokens", - type=int, - metavar="N", - default=50000, - help="maximum samples for encoder speech input ", - ) - parser.add_argument( - "--max-speech-positions", - type=int, - metavar="N", - default=400, - help="maximum tokens for per encoder text input ", - ) - - parser.add_argument( - "--max-sample-size", - type=int, - metavar="N", - default=32000, - help="max sample size to crop to for batching (unsupervised speech) ", - ) - parser.add_argument( - "--min-sample-size", - type=int, - metavar="N", - default=4000, - help="min sample size to crop to for batching (unsupervised speech) ", - ) - - # set mini-batch ratio for different modalities/subtasks - # s2p - parser.add_argument( - "--supervised-speech-sample-ratio", - default="1", - type=str, - metavar="N", - help="Multiple Ratio for speech dataset with transcripts ", - ) - # s2t - parser.add_argument( - "--supervised-speech-s2s-sample-ratio", - default="1", - type=str, - metavar="N", - help="Multiple Ratio for speech dataset with transcripts ", - ) - # ssl - parser.add_argument( - "--unsupervised-speech-sample-ratio", - default="1", - type=str, - metavar="N", - help="Multiple Ratio for speech dataset without transcripts ", - ) - # t2t with monolingual data (masking) - parser.add_argument( - "--text-sample-ratio", - default="1", - type=str, - metavar="N", - help="Multiple Ratio for text set ", - ) - # t2t with parallel data (no masking) - parser.add_argument( - "--bitext-sample-ratio", - default="1", - type=str, - metavar="N", - help="Multiple Ratio for text set (bitext) ", - ) - # train_subset = "train", 'valid' or so - # parallel data is loaded according to string lang_pairs and lang_pairs_no_mask from args.data - # (un)supervised speech is loaded from args.(un)sup_speech_{train,valid}_subset - parser.add_argument( - "--sup-speech-data", default="", help="path to supervised speech data" - ) - parser.add_argument( - "--sup-speech-train-subset", - default="", - help="supervised speech training subsets", - ) - parser.add_argument( - "--sup-speech-valid-subset", - default="", - help="supervised speech validation subsets", - ) - parser.add_argument( - "--config-yaml", - default="config.yaml", - help="supervised speech configuration yaml file", - ) - parser.add_argument( - "--sup-speech-s2s-data", default="", help="path to supervised speech data" - ) - parser.add_argument( - "--sup-speech-s2s-train-subset", - default="", - help="supervised speech training subsets", - ) - parser.add_argument( - "--sup-speech-s2s-valid-subset", - default="", - help="supervised speech validation subsets", - ) - parser.add_argument( - "--config-s2s-yaml", - default="config.yaml", - help="supervised speech configuration yaml file", - ) - parser.add_argument( - "--unsup-speech-train-data", - default="", - help="path to unsupervised speech training data (tsv)", - ) - parser.add_argument( - "--unsup-speech-valid-data", - default="", - help="path to unsupervised speech valid data (tsv)", - ) - parser.add_argument( - "--sample-rate", - type=int, - metavar="N", - default=16000, - help="input audio sampling rate", - ) - parser.add_argument( - "--no-emb-update-unsup", - default=False, - action="store_true", - help="no update for output embedding during unsupervised_speech mode", - ) - parser.add_argument("--same-data-update", default=False, action="store_true") - - # used for sup_speech_ali - parser.add_argument( - "--use-sup-speech-ctc", - default=False, - action="store_true", - help="use speech_sup_ctc instead of speech_sup_ali", - ) - - @classmethod - def setup_task(cls, args, **kwargs): - """Setup the task.""" - paths = args.data.split(":") - assert len(paths) > 0 - src_dict = Dictionary.load( - os.path.join(paths[0], "src_dict.txt") - ) # assume all languages share a source dictionary - tgt_dict = Dictionary.load( - os.path.join(paths[0], "tgt_dict.txt") - ) # assume all languages share a target dictionary - - lang_pairs = args.lang_pairs + "," + args.lang_pairs_bitext - lang_pairs = re.sub(",$", "", re.sub("^,", "", lang_pairs)) - if lang_pairs != "": - src_langs = [lp.split("-")[0] for lp in lang_pairs.split(",")] - tgt_langs = [lp.split("-")[1] for lp in lang_pairs.split(",")] - else: - src_langs = [] - tgt_langs = [] - - if args.add_src_lang_token: - for lang in src_langs: - assert ( - src_dict.index(PairedDenoisingTask.LANG_TAG_TEMPLATE.format(lang)) - != src_dict.unk() - ) - if args.add_tgt_lang_token: - for lang in tgt_langs: - assert ( - tgt_dict.index(PairedDenoisingTask.LANG_TAG_TEMPLATE.format(lang)) - != tgt_dict.unk() - ) - - logger.info("source dictionary: {} types".format(len(src_dict))) - logger.info("target dictionary: {} types".format(len(tgt_dict))) - if not hasattr(args, "shuffle_instance"): - args.shuffle_instance = False - return cls(args, src_dict, tgt_dict) - - def __init__(self, args, src_dict, tgt_dict): - super().__init__(args, src_dict, tgt_dict) - self.data_cfg = S2TJointDataConfig( - Path(args.sup_speech_data) / args.config_yaml - ) - logger.info( - f"load supervised speech data configure from {Path(args.sup_speech_data) / args.config_yaml}" - ) - self.data_s2s_cfg = ( - S2TJointDataConfig(Path(args.sup_speech_s2s_data) / args.config_s2s_yaml) - if args.sup_speech_s2s_train_subset != "" - else None - ) - if self.data_s2s_cfg is not None: - logger.info( - f"load supervised sequece to sequence speech data configure from {Path(args.sup_speech_s2s_data) / args.config_yaml}" - ) - - def parse_data_ratio(sample_ratio): - ratios = sample_ratio.split(",") - if len(ratios) == 1: - return [float(ratios[0])] - epoch_ratios = [] - for item in ratios: - ep, r = item.split(":") - ep = int(ep) - r = float(r) - assert ep > 0 # epoch is 1 based - assert ep >= len(epoch_ratios) - - if len(epoch_ratios) == 0: - epoch_ratios.append( - r - ) # epoch_ratios[0] is not used, but we still set it to the first value to make thing simple. - while len(epoch_ratios) < ep: - epoch_ratios.append(epoch_ratios[-1]) - epoch_ratios.append(r) - return epoch_ratios - - self.sup_ratio = parse_data_ratio(args.supervised_speech_sample_ratio) - self.sup_s2s_ratio = parse_data_ratio(args.supervised_speech_s2s_sample_ratio) - self.text_ratio = parse_data_ratio(args.text_sample_ratio) - self.bitext_ratio = parse_data_ratio(args.bitext_sample_ratio) - self.unsup_ratio = parse_data_ratio(args.unsupervised_speech_sample_ratio) - self.sample_mode = None - - def build_model(self, args): - args.input_feat_per_channel = self.data_cfg.input_feat_per_channel - args.input_channels = self.data_cfg.input_channels - return super().build_model(args) - - def build_tokenizer(self, data_cfg, msg=""): - logger.info(f"pre-tokenizer {msg}: {data_cfg.pre_tokenizer}") - return encoders.build_tokenizer(Namespace(**data_cfg.pre_tokenizer)) - - def build_bpe(self, data_cfg, msg=""): - logger.info(f"tokenizer {msg}: {data_cfg.bpe_tokenizer}") - return encoders.build_bpe(Namespace(**data_cfg.bpe_tokenizer)) - - @classmethod - def resolve_data_type(cls, split, use_sup_speech_ctc): - if len(split.split("_")) == 1: - # default case, train or valid - is_train = split - dtype = "text" - else: - is_train, dtype = split.split("_", 1) - is_train = True if is_train == "train" else False - if dtype == "sup_speech": - dtype = "sup_speech_ctc" if use_sup_speech_ctc else "sup_speech_ali" - assert dtype in ( - "text", - "bitext", - "sup_speech_ali", - "sup_speech_s2s", - "unsup_speech", - "sup_speech_ctc", - ) - return is_train, dtype - - def create_modalitydatasetitem(self, dtype, dataset): - dsitem = None - if dtype in ("text", "bitext"): - dsitem = ModalityDatasetItem( - dtype, - dataset, - (self.args.max_source_positions, self.args.max_target_positions), - self.args.max_text_tokens, - self.args.batch_size, - ) - elif dtype in ("sup_speech_ctc", "sup_speech_ali", "sup_speech_s2s"): - dsitem = ModalityDatasetItem( - dtype, - dataset, - (self.args.max_speech_positions, self.args.max_target_positions), - self.args.max_speech_tokens, - self.args.batch_size, - ) - elif dtype == "unsup_speech": - dsitem = ModalityDatasetItem( - dtype, dataset, 1e8, self.args.max_speech_tokens, self.args.batch_size - ) - else: - raise ValueError(f"{dtype} is not supported") - return dsitem - - def load_dataset(self, split, epoch=1, combine=False, **kwargs): - def _get_sup_src_tgt_dict(src_dict, tgt_dict, use_s2s_sup_decoder): - if use_s2s_sup_decoder: - return None, tgt_dict - # use src_dict as tgt_dict here, since we use source dictionary as target for forcealignment - return None, src_dict - - is_train, dtype = self.resolve_data_type(split, self.args.use_sup_speech_ctc) - - # Note we use --add-tgt-lang-token instead of data_cfg.prepend_tgt_lang_tag_no_change to set target language tag in the text dataset - # Verify add_tgt_lang_token and prepend_tgt_lang_tag_no_change are same - - # Note we use --multilang-sampling-alpha instead of data_cfg.sampling_text_alpha to set text data sampling - if is_train: - msets = [] - # train split, load everything into one - if self.lang_pairs != "": - text_dataset = self.load_dataset_only( - "train", self.lang_pairs, epoch=epoch, combine=combine - ) - dsitem = self.create_modalitydatasetitem("text", text_dataset) - msets.append(dsitem) - if self.lang_pairs_bitext != "": # load bitext - bitext_dataset = self.load_dataset_only( - "train_bitext", - self.lang_pairs_bitext, - do_mask=False, - epoch=epoch, - combine=combine, - ) - dsitem = self.create_modalitydatasetitem("bitext", bitext_dataset) - msets.append(dsitem) - if self.args.sup_speech_train_subset != "": - pre_tokenizer = self.build_tokenizer(self.data_cfg) - bpe_tokenizer = self.build_bpe(self.data_cfg) - - append_eos = True - sup_speech_type = "sup_speech_ali" - if self.args.use_sup_speech_ctc: - # CTC mode - sup_speech_type = "sup_speech_ctc" - append_eos = False # CTC doesn't need eos in the target - - src_dict, tgt_dict = _get_sup_src_tgt_dict( - self.src_dict, self.tgt_dict, False - ) - sup_speech_dataset = SpeechToTextJointDatasetCreator.from_tsv( - self.args.sup_speech_data, - self.data_cfg, - self.args.sup_speech_train_subset, - tgt_dict=tgt_dict, - src_dict=src_dict, - pre_tokenizer=pre_tokenizer, - bpe_tokenizer=bpe_tokenizer, - src_pre_tokenizer=None, - src_bpe_tokenizer=None, - is_train_split=is_train, - epoch=epoch, - seed=self.args.seed, - append_eos=append_eos, - ) - dsitem = self.create_modalitydatasetitem( - sup_speech_type, sup_speech_dataset - ) - msets.append(dsitem) - - if self.args.sup_speech_s2s_train_subset != "": - pre_tokenizer = self.build_tokenizer(self.data_s2s_cfg, msg="(s2s)") - bpe_tokenizer = self.build_bpe(self.data_s2s_cfg, msg="(s2s)") - - # make sure self.data_cfg.prepend_tgt_lang_tag_no_change == self.args.add_tgt_lang_token - src_dict, tgt_dict = _get_sup_src_tgt_dict( - self.src_dict, self.tgt_dict, True - ) - sup_speech_s2s_dataset = SpeechToTextJointDatasetCreator.from_tsv( - self.args.sup_speech_s2s_data, - self.data_s2s_cfg, - self.args.sup_speech_s2s_train_subset, - tgt_dict=tgt_dict, - src_dict=src_dict, - pre_tokenizer=pre_tokenizer, - bpe_tokenizer=bpe_tokenizer, - src_pre_tokenizer=None, - src_bpe_tokenizer=None, - is_train_split=is_train, - epoch=epoch, - seed=self.args.seed, - ) - dsitem = self.create_modalitydatasetitem( - "sup_speech_s2s", sup_speech_s2s_dataset - ) - msets.append(dsitem) - if self.args.unsup_speech_train_data != "": - unsup_speech_dataset = FileAudioDatasetWrapper( - self.args.unsup_speech_train_data, - self.args.sample_rate, - max_sample_size=self.args.max_sample_size, - min_sample_size=self.args.min_sample_size, - normalize=False, - ) - dsitem = self.create_modalitydatasetitem( - "unsup_speech", unsup_speech_dataset - ) - msets.append(dsitem) - - pre_train_dataset = MultiModalityDataset(msets) - self.datasets[split] = pre_train_dataset - else: # validation split, load them for each type of data - if dtype == "text": - text_dataset = self.load_dataset_only( - split, self.lang_pairs, epoch=epoch, combine=combine - ) - dsitem = self.create_modalitydatasetitem("text", text_dataset) - self.datasets[split] = MultiModalityDataset([dsitem]) - elif dtype == "bitext": - bitext_dataset = self.load_dataset_only( - split, - self.lang_pairs_bitext, - do_mask=False, - epoch=epoch, - combine=combine, - ) - dsitem = self.create_modalitydatasetitem("bitext", bitext_dataset) - self.datasets[split] = MultiModalityDataset([dsitem]) - - elif dtype in ("sup_speech_ctc", "sup_speech_ali"): - assert self.args.sup_speech_valid_subset != "" - pre_tokenizer = self.build_tokenizer(self.data_cfg) - bpe_tokenizer = self.build_bpe(self.data_cfg) - append_eos = True - if dtype == "sup_speech_ctc": - # CTC mode - append_eos = False # CTC doesn't need eos - assert self.args.use_sup_speech_ctc - - datasets = [] - for split_name in self.args.sup_speech_valid_subset.split(","): - src_dict, tgt_dict = _get_sup_src_tgt_dict( - self.src_dict, self.tgt_dict, False - ) - datasets.append( - SpeechToTextJointDatasetCreator.from_tsv( - self.args.sup_speech_data, - self.data_cfg, - split_name, - tgt_dict=tgt_dict, - src_dict=src_dict, - pre_tokenizer=pre_tokenizer, - bpe_tokenizer=bpe_tokenizer, - src_pre_tokenizer=None, - src_bpe_tokenizer=None, - is_train_split=is_train, - epoch=epoch, - seed=self.args.seed, - append_eos=append_eos, - ) - ) - - dset = datasets[0] if len(datasets) == 1 else ConcatDataset(datasets) - dsitem = self.create_modalitydatasetitem(dtype, dset) - self.datasets[split] = MultiModalityDataset([dsitem]) - - elif dtype == "sup_speech_s2s": - assert self.args.sup_speech_s2s_valid_subset != "" - pre_tokenizer = self.build_tokenizer(self.data_s2s_cfg) - bpe_tokenizer = self.build_bpe(self.data_s2s_cfg) - datasets = [] - for split_name in self.args.sup_speech_s2s_valid_subset.split(","): - src_dict, tgt_dict = _get_sup_src_tgt_dict( - self.src_dict, self.tgt_dict, True - ) - datasets.append( - SpeechToTextJointDatasetCreator.from_tsv( - self.args.sup_speech_s2s_data, - self.data_s2s_cfg, - split_name, - tgt_dict=tgt_dict, - src_dict=src_dict, - pre_tokenizer=pre_tokenizer, - bpe_tokenizer=bpe_tokenizer, - src_pre_tokenizer=None, - src_bpe_tokenizer=None, - is_train_split=is_train, - epoch=epoch, - seed=self.args.seed, - ) - ) - - dset = datasets[0] if len(datasets) == 1 else ConcatDataset(datasets) - dsitem = self.create_modalitydatasetitem("sup_speech_s2s", dset) - self.datasets[split] = MultiModalityDataset([dsitem]) - elif dtype == "unsup_speech": - assert self.args.unsup_speech_valid_data != "" - unsup_speech_dataset = FileAudioDatasetWrapper( - self.args.unsup_speech_valid_data, - self.args.sample_rate, - max_sample_size=self.args.max_sample_size, - min_sample_size=self.args.min_sample_size, - normalize=False, - ) - dsitem = self.create_modalitydatasetitem( - "unsup_speech", unsup_speech_dataset - ) - self.datasets[split] = MultiModalityDataset([dsitem]) - else: - raise ValueError(f"Unsupported type {dtype}") - - def get_sample_ratio(self, epoch): - sup_ratio = ( - self.sup_ratio[epoch] if len(self.sup_ratio) > epoch else self.sup_ratio[-1] - ) - sup_s2s_ratio = ( - self.sup_s2s_ratio[epoch] - if len(self.sup_s2s_ratio) > epoch - else self.sup_s2s_ratio[-1] - ) - unsup_ratio = ( - self.unsup_ratio[epoch] - if len(self.unsup_ratio) > epoch - else self.unsup_ratio[-1] - ) - text_ratio = ( - self.text_ratio[epoch] - if len(self.text_ratio) > epoch - else self.text_ratio[-1] - ) - bitext_ratio = ( - self.bitext_ratio[epoch] - if len(self.bitext_ratio) > epoch - else self.bitext_ratio[-1] - ) - return text_ratio, bitext_ratio, sup_ratio, sup_s2s_ratio, unsup_ratio - - def get_batch_iterator( - self, - dataset, - max_tokens=None, - max_sentences=None, - max_positions=None, - ignore_invalid_inputs=False, - required_batch_size_multiple=1, - seed=1, - num_shards=1, - shard_id=0, - num_workers=0, - epoch=0, - data_buffer_size=0, - disable_iterator_cache=False, - skip_remainder_batch=False, - grouped_shuffling=False, - update_epoch_batch_itr=False, - ): - - assert isinstance(dataset, MultiModalityDataset) - if len(dataset.id_to_mode) == 1: - max_positions = dataset.max_positions[0] - max_tokens = dataset.max_tokens[0] - max_sentences = dataset.max_sentences[0] - return super().get_batch_iterator( - dataset, - max_tokens, - max_sentences, - max_positions, - ignore_invalid_inputs, - required_batch_size_multiple, - seed, - num_shards, - shard_id, - num_workers, - epoch, - data_buffer_size, - disable_iterator_cache, - skip_remainder_batch=skip_remainder_batch, - ) - - mult_ratio = [] - ( - text_ratio, - bitext_ratio, - sup_ratio, - sup_s2s_ratio, - unsup_ratio, - ) = self.get_sample_ratio(epoch) - for mode in dataset.id_to_mode: - if mode in ("sup_speech_ctc", "sup_speech_ali"): - mult_ratio.append(sup_ratio) - elif mode == "sup_speech_s2s": - mult_ratio.append(sup_s2s_ratio) - elif mode == "text": - mult_ratio.append(text_ratio) - elif mode == "bitext": - mult_ratio.append(bitext_ratio) - elif mode == "unsup_speech": - mult_ratio.append(unsup_ratio) - - # initialize the dataset with the correct starting epoch - dataset.set_epoch(epoch) - - batch_samplers = dataset.get_batch_samplers( - mult_ratio, required_batch_size_multiple, seed - ) - - # return a reusable, sharded iterator - epoch_iter = GroupedEpochBatchIterator( - dataset=dataset, - collate_fn=dataset.collater, - batch_samplers=batch_samplers, - seed=seed, - num_shards=num_shards, - shard_id=shard_id, - num_workers=num_workers, - epoch=epoch, - mult_rate=max(self.args.update_freq) if self.args.same_data_update else 1, - buffer_size=data_buffer_size, - skip_remainder_batch=skip_remainder_batch, - ) - self.dataset_to_epoch_iter[dataset] = {} # refresh it every epoch - return epoch_iter diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/tasks/speech_text_joint.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/tasks/speech_text_joint.py deleted file mode 100644 index c94dda28aa76e803eade5915e9eb0e69e542778d..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_text_joint_to_text/tasks/speech_text_joint.py +++ /dev/null @@ -1,377 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. -import logging -import os -from argparse import Namespace -from pathlib import Path - -import torch -from fairseq.data import ( - ConcatDataset, - Dictionary, - ResamplingDataset, - TransformEosLangPairDataset, - encoders, -) -from fairseq.data.audio.multi_modality_dataset import ( - LangPairMaskDataset, - ModalityDatasetItem, - MultiModalityDataset, -) -from fairseq.data.audio.speech_to_text_dataset import ( - SpeechToTextDataset, - SpeechToTextDatasetCreator, -) -from fairseq.data.audio.speech_to_text_joint_dataset import ( - S2TJointDataConfig, - SpeechToTextJointDatasetCreator, -) -from fairseq.data.iterators import GroupedEpochBatchIterator -from fairseq.tasks import register_task -from fairseq.tasks.speech_to_text import SpeechToTextTask -from fairseq.tasks.translation import load_langpair_dataset - -logger = logging.getLogger(__name__) -LANG_TAG_TEMPLATE = "" - - -@register_task("speech_text_joint_to_text") -class SpeechTextJointToTextTask(SpeechToTextTask): - """ - Task for joint training speech and text to text. - """ - - @classmethod - def add_args(cls, parser): - """Add task-specific arguments to the parser.""" - super(SpeechTextJointToTextTask, cls).add_args(parser) - ### - parser.add_argument( - "--parallel-text-data", - default="", - help="path to parallel text data directory", - ) - parser.add_argument( - "--max-tokens-text", - type=int, - metavar="N", - help="maximum tokens for encoder text input ", - ) - parser.add_argument( - "--max-positions-text", - type=int, - metavar="N", - default=400, - help="maximum tokens for per encoder text input ", - ) - parser.add_argument( - "--langpairs", - default=None, - metavar="S", - help='language pairs for text training, separated with ","', - ) - parser.add_argument( - "--speech-sample-ratio", - default=1, - type=float, - metavar="N", - help="Multiple Ratio for speech dataset with transcripts ", - ) - parser.add_argument( - "--text-sample-ratio", - default=1, - type=float, - metavar="N", - help="Multiple Ratio for text set ", - ) - parser.add_argument( - "--update-mix-data", - action="store_true", - help="use mixed data in one update when update-freq > 1", - ) - parser.add_argument( - "--load-speech-only", action="store_true", help="load speech data only", - ) - parser.add_argument( - "--mask-text-ratio", - type=float, - metavar="V", - default=0.0, - help="mask V source tokens for text only mode", - ) - parser.add_argument( - "--mask-text-type", - default="random", - choices=["random", "tail"], - help="mask text typed", - ) - parser.add_argument( - "--noise-token", - default="", - help="noise token for masking src text tokens if mask-text-ratio > 0", - ) - parser.add_argument( - "--infer-target-lang", - default="", - metavar="S", - help="target language for inference", - ) - - def __init__(self, args, src_dict, tgt_dict, infer_tgt_lang_id=None): - super().__init__(args, tgt_dict) - self.src_dict = src_dict - self.data_cfg = S2TJointDataConfig(Path(args.data) / args.config_yaml) - assert self.tgt_dict.pad() == self.src_dict.pad() - assert self.tgt_dict.eos() == self.src_dict.eos() - self.speech_only = args.load_speech_only - self._infer_tgt_lang_id = infer_tgt_lang_id - - @classmethod - def setup_task(cls, args, **kwargs): - """Setup the task (e.g., load dictionaries).""" - data_cfg = S2TJointDataConfig(Path(args.data) / args.config_yaml) - tgt_dict_path = Path(args.data) / data_cfg.vocab_filename - src_dict_path = Path(args.data) / data_cfg.src_vocab_filename - if (not os.path.isfile(src_dict_path)) or (not os.path.isfile(tgt_dict_path)): - raise FileNotFoundError("Dict not found: {}".format(args.data)) - src_dict = Dictionary.load(src_dict_path.as_posix()) - tgt_dict = Dictionary.load(tgt_dict_path.as_posix()) - - print("| src dictionary: {} types".format(len(src_dict))) - print("| tgt dictionary: {} types".format(len(tgt_dict))) - - if args.parallel_text_data != "": - if not os.path.isabs(args.parallel_text_data): - args.parallel_text_data = os.path.join( - args.data, args.parallel_text_data - ) - - if args.langpairs is None: - raise Exception( - "Could not infer language pair, please provide it explicitly" - ) - infer_tgt_lang_id = None - if args.infer_target_lang != "" and data_cfg.prepend_tgt_lang_tag_no_change: - tgt_lang_tag = SpeechToTextDataset.LANG_TAG_TEMPLATE.format( - args.infer_target_lang - ) - infer_tgt_lang_id = tgt_dict.index(tgt_lang_tag) - assert infer_tgt_lang_id != tgt_dict.unk() - return cls(args, src_dict, tgt_dict, infer_tgt_lang_id=infer_tgt_lang_id) - - def load_langpair_dataset( - self, prepend_tgt_lang_tag=False, sampling_alpha=1.0, epoch=0 - ): - lang_pairs = [] - text_dataset = None - split = "train" - for lp in self.args.langpairs.split(","): - src, tgt = lp.split("-") - text_dataset = load_langpair_dataset( - self.args.parallel_text_data, - split, - src, - self.src_dict, - tgt, - self.tgt_dict, - combine=True, - dataset_impl=None, - upsample_primary=1, - left_pad_source=False, - left_pad_target=False, - max_source_positions=self.args.max_positions_text, - max_target_positions=self.args.max_target_positions, - load_alignments=False, - truncate_source=False, - ) - if prepend_tgt_lang_tag: - # TODO - text_dataset = TransformEosLangPairDataset( - text_dataset, - src_eos=self.src_dict.eos(), - tgt_bos=self.tgt_dict.eos(), # 'prev_output_tokens' starts with eos - new_tgt_bos=self.tgt_dict.index(LANG_TAG_TEMPLATE.format(tgt)), - ) - lang_pairs.append(text_dataset) - if len(lang_pairs) > 1: - if sampling_alpha != 1.0: - size_ratios = SpeechToTextDatasetCreator.get_size_ratios( - self.args.langpairs.split(","), - [len(s) for s in lang_pairs], - alpha=sampling_alpha, - ) - lang_pairs = [ - ResamplingDataset(d, size_ratio=r, epoch=epoch, replace=(r >= 1.0)) - for d, r in zip(lang_pairs, size_ratios) - ] - return ConcatDataset(lang_pairs) - return text_dataset - - def inference_step( - self, generator, models, sample, prefix_tokens=None, constraints=None - ): - with torch.no_grad(): - return generator.generate( - models, - sample, - prefix_tokens=prefix_tokens, - constraints=constraints, - bos_token=self._infer_tgt_lang_id, - ) - - def build_src_tokenizer(self, args): - logger.info(f"src-pre-tokenizer: {self.data_cfg.src_pre_tokenizer}") - return encoders.build_tokenizer(Namespace(**self.data_cfg.src_pre_tokenizer)) - - def build_src_bpe(self, args): - logger.info(f"tokenizer: {self.data_cfg.src_bpe_tokenizer}") - return encoders.build_bpe(Namespace(**self.data_cfg.src_bpe_tokenizer)) - - def load_dataset(self, split, epoch=1, combine=False, **kwargs): - """Load a given dataset split. - - Args: - split (str): name of the split (e.g., train, valid, test) - """ - is_train_split = split.startswith("train") - pre_tokenizer = self.build_tokenizer(self.args) - bpe_tokenizer = self.build_bpe(self.args) - src_pre_tokenizer = self.build_src_tokenizer(self.args) - src_bpe_tokenizer = self.build_src_bpe(self.args) - ast_dataset = SpeechToTextJointDatasetCreator.from_tsv( - self.args.data, - self.data_cfg, - split, - self.tgt_dict, - src_dict=None if self.speech_only else self.src_dict, - pre_tokenizer=pre_tokenizer, - bpe_tokenizer=bpe_tokenizer, - src_pre_tokenizer=src_pre_tokenizer, - src_bpe_tokenizer=src_bpe_tokenizer, - is_train_split=is_train_split, - epoch=epoch, - seed=self.args.seed, - ) - noise_token_id = -1 - text_dataset = None - if self.args.parallel_text_data != "" and is_train_split: - text_dataset = self.load_langpair_dataset( - self.data_cfg.prepend_tgt_lang_tag_no_change, 1.0, epoch=epoch, - ) - if self.args.mask_text_ratio > 0: - # add mask - noise_token_id = ( - self.src_dict.unk() - if self.args.noise_token == "" - else self.src_dict.index(self.args.noise_token) - ) - text_dataset = LangPairMaskDataset( - text_dataset, - src_bos=self.src_dict.bos(), - src_eos=self.src_dict.eos(), - noise_id=noise_token_id, - mask_ratio=self.args.mask_text_ratio, - mask_type=self.args.mask_text_type, - ) - - if text_dataset is not None: - mdsets = [ - ModalityDatasetItem( - "sup_speech", - ast_dataset, - (self.args.max_source_positions, self.args.max_target_positions), - self.args.max_tokens, - self.args.batch_size, - ), - ModalityDatasetItem( - "text", - text_dataset, - (self.args.max_positions_text, self.args.max_target_positions), - self.args.max_tokens_text - if self.args.max_tokens_text is not None - else self.args.max_tokens, - self.args.batch_size, - ), - ] - ast_dataset = MultiModalityDataset(mdsets) - self.datasets[split] = ast_dataset - - @property - def target_dictionary(self): - """Return the :class:`~fairseq.data.Dictionary` for the language - model.""" - return self.tgt_dict - - @property - def source_dictionary(self): - """Return the source :class:`~fairseq.data.Dictionary` (if applicable - for this task).""" - return None if self.speech_only else self.src_dict - - def get_batch_iterator( - self, - dataset, - max_tokens=None, - max_sentences=None, - max_positions=None, - ignore_invalid_inputs=False, - required_batch_size_multiple=1, - seed=1, - num_shards=1, - shard_id=0, - num_workers=0, - epoch=0, - data_buffer_size=0, - disable_iterator_cache=False, - skip_remainder_batch=False, - grouped_shuffling=False, - update_epoch_batch_itr=False, - ): - - if not isinstance(dataset, MultiModalityDataset): - return super(SpeechTextJointToTextTask, self).get_batch_iterator( - dataset, - max_tokens, - max_sentences, - max_positions, - ignore_invalid_inputs, - required_batch_size_multiple, - seed, - num_shards, - shard_id, - num_workers, - epoch, - data_buffer_size, - disable_iterator_cache, - skip_remainder_batch=skip_remainder_batch, - update_epoch_batch_itr=update_epoch_batch_itr, - ) - - mult_ratio = [self.args.speech_sample_ratio, self.args.text_sample_ratio] - assert len(dataset.datasets) == 2 - - # initialize the dataset with the correct starting epoch - dataset.set_epoch(epoch) - - batch_samplers = dataset.get_batch_samplers( - mult_ratio, required_batch_size_multiple, seed - ) - - # return a reusable, sharded iterator - epoch_iter = GroupedEpochBatchIterator( - dataset=dataset, - collate_fn=dataset.collater, - batch_samplers=batch_samplers, - seed=seed, - num_shards=num_shards, - shard_id=shard_id, - num_workers=num_workers, - epoch=epoch, - mult_rate=1 if self.args.update_mix_data else max(self.args.update_freq), - buffer_size=data_buffer_size, - skip_remainder_batch=skip_remainder_batch, - ) - self.dataset_to_epoch_iter[dataset] = {} # refresh it every epoch - return epoch_iter diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/README.md b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/README.md deleted file mode 100644 index f03f6a32f8b3a0070440918b775f18e712480885..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Speech to speech translation (S2ST) - -We provide the implementation and resources for the following work on speech-to-speech translation (S2ST): - -* [Direct speech-to-speech translation with discrete units (Lee et al. 2021)](docs/direct_s2st_discrete_units.md) -* [Textless Speech-to-Speech Translation on Real Data (Lee et al. 2021)](docs/textless_s2st_real_data.md) -* [Enhanced Direct Speech-to-Speech Translation Using Self-supervised Pre-training and Data Augmentation](docs/enhanced_direct_s2st_discrete_units.md) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/__init__.py deleted file mode 100644 index 6264236915a7269a4d920ee8213004374dd86a9a..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/benchmarking/README.md b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/benchmarking/README.md deleted file mode 100644 index c62fe12963a20a09078d3c79c65f3ebedf1aa3f9..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/benchmarking/README.md +++ /dev/null @@ -1,31 +0,0 @@ -# Benchmarking - -## Overview - -The goal of this framework is to support benchmarking various speech to speech translation(S2ST) models in terms of runtime, max-memory consumption and total number of floating point operations(FLOPS). It is a generic framework and can be easily extended to support any fairseq models. To accurately benchmark the performance, core inference modules are re-implemented based on fairseq_cli/generate.py (core.py/Processing) and examples/speech_to_text/generate_waveform.py(core.py/SpeechGeneration. To ensure that the end to end models and cascaded models are compared fairly, for cascaded models we only consider the performance metrics for model inference at all stages ignoring any intermediate data and io processing consumption. We run all the benchmarking runs on CPU as it is generally used in production environment and also due to lack of good benchmarking library support for GPUs. - -1. Runtime: Average time in seconds to run model inference on an example from a given dataset. We use [timeit](https://docs.python.org/3/library/timeit.html) library to measure the runtime. -2. Max memory: Maximum memory in MiB averaged over by running the model inference on all examples from the given dataset. We use [memory_profiler](https://pypi.org/project/memory-profiler/) library to gather memory footprints for a code snippet and find the maximum to get the max memory used by the code. For cascaded models, we find the max of all stages to get the overall max_memory footprint. -3. FLOPS: We compute the average number of floating point operations needed to run model inference for an example from the given dataset. We use [PAPI library](http://www.bnikolic.co.uk/blog/python/flops/2019/10/01/pytorch-count-flops.html) to benchmark the number of flops. - -## CLI Commands - -```{python} -CUBLAS_WORKSPACE_CONFIG=:4096:8 python examples/speech_to_speech/benchmarking/get_metrics.py ‘’ --config $config -``` - - -## Note: - -1. The npy dataset is a list of samples saved as a .npy file. Each sample is a dictionary with id, net_input. -2. The raw dataset is a list of raw audio paths similar to wav2vec2 input tsv file - -```{python} -sample: { - "id": xx, - "net_input": { - "src_tokens": torch.tensor([]), - "src_lengths": torch.tensor([]) - } -} -``` diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/benchmarking/configs/2StageS2ST.yaml b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/benchmarking/configs/2StageS2ST.yaml deleted file mode 100644 index 11deb42e7d63cf4783d19a6fe17fb8b8cd483aff..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/benchmarking/configs/2StageS2ST.yaml +++ /dev/null @@ -1,19 +0,0 @@ -general: - dataset_path: $npy_dataset - cpu: True - model_type: 2StageS2ST - dataset_size: 1 - -stage1: - data: $data_bin_stage1 - task: speech_to_text - path: $checkpoint_stage1 - config_yaml: config.yaml - max_len_a: 2 - max_len_b: 500 - -stage2: - data: $data_bin_stage2 - task: text_to_speech - path: $checkpoint_stage2 - config_yaml: config.yaml diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/benchmarking/configs/3StageS2ST.yaml b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/benchmarking/configs/3StageS2ST.yaml deleted file mode 100644 index 96381361508ce5d459f9020ccaf2b18a2fc18aaa..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/benchmarking/configs/3StageS2ST.yaml +++ /dev/null @@ -1,28 +0,0 @@ -general: - dataset_path: $npy_dataset - cpu: True - model_type: 3StageS2ST - max_len_a: 2 - max_len_b: 500 - dataset_size: 1 - -stage1: - data: $data_bin_stage1 - task: speech_to_text - path: $checkpoint_stage1 - config_yaml: config.yaml - max_len_a: 2 - max_len_b: 500 - -stage2: - data: $data_bin_stage2 - task: translation - path: $checkpoint_stage2 - config_yaml: config.yaml - - -stage2: - data: $data_bin_stage3 - task: text_to_speech - path: $checkpoint_stage3 - config_yaml: config.yaml diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/benchmarking/configs/DirectS2U.yaml b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/benchmarking/configs/DirectS2U.yaml deleted file mode 100644 index 96264cec685ba9d0371318866fcc6c82fffdc02d..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/benchmarking/configs/DirectS2U.yaml +++ /dev/null @@ -1,22 +0,0 @@ -general: - dataset_path: $npy_dataset_path - cpu: True - model_type: S2UT - dataset_size: 5 - dump_speech_waveforms_dir: $dump_waveforms_dir_path - -stage1: - data: $data_bin - task: speech_to_speech - path: $checkpoint - config_yaml: config.yaml - max_len_b: 100000 - beam: 10 - target_is_code: True - max_target_positions: 3000 - target_code_size: 100 - -stage2: - vocoder: $vocoder_path - vocoder_cfg: $vocoder_cfg_json - dur_prediction: True diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/benchmarking/configs/S2T.yaml b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/benchmarking/configs/S2T.yaml deleted file mode 100644 index 3a106a0441007331e926870a7310e114daf598a0..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/benchmarking/configs/S2T.yaml +++ /dev/null @@ -1,13 +0,0 @@ -general: - dataset_path: $npy_dataset - cpu: True - model_type: S2T - dataset_size: 1 - -stage1: - data: $data_bin - task: speech_to_text - path: $checkpoint - config_yaml: config.yaml - max_len_a: 2 - max_len_b: 500 diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/benchmarking/core.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/benchmarking/core.py deleted file mode 100644 index 9b10b92c078bb956c30ef3dd3d3711c85672d58f..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/benchmarking/core.py +++ /dev/null @@ -1,489 +0,0 @@ -import ast -import json -import logging -import timeit -from argparse import Namespace - -import soundfile as sf -import torch -from examples.hubert.simple_kmeans.dump_hubert_feature import HubertFeatureReader -from examples.hubert.simple_kmeans.dump_km_label import ApplyKmeans -from fairseq import checkpoint_utils, tasks, utils -from fairseq.data import data_utils as fairseq_data_utils -from fairseq.dataclass.utils import convert_namespace_to_omegaconf -from fairseq.models.text_to_speech.vocoder import CodeHiFiGANVocoder -from fairseq_cli.generate import get_symbols_to_strip_from_output -from memory_profiler import memory_usage -from pypapi import events -from pypapi import papi_high as high -from torch import nn - -logging.basicConfig() -logging.root.setLevel(logging.INFO) -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - - -torch.manual_seed(1) -torch.set_deterministic(True) - - -class BenchmarkingBase(nn.Module): - def __init__(self): - nn.Module.__init__(self) - self.s2x_task = None - - def warm_up(self, sample, repeat): - """Warm up the model""" - for _i in range(repeat): - self.forward(sample) - logger.info(f"Model warmed up by running inference {repeat} times") - - def benchmark_run_time(self, dataset, repeat): - """Benchmark average runtime for the model by calling benchmark_run_time_single_sample function""" - logger.info("Starting run time benchmarking") - time_elapsed = 0 - for i, sample in enumerate(dataset): - time_elapsed += self.benchmark_run_time_single_sample(sample, repeat=repeat) - if i % 100 == 0: - logger.info(f"Benchmarked run time for {i}/{len(dataset)} samples") - total_time_elapsed = time_elapsed / len(dataset) - return total_time_elapsed - - def benchmark_run_time_single_sample(self, sample, repeat): - """Benchmark average runtime for a single sample using timeit library. Units are seconds""" - timer = timeit.Timer(lambda: self.forward(sample)) - time_elapsed = timer.timeit(repeat) - return time_elapsed / repeat - - def count_flops( - self, - dataset, - repeat, - ): - """Use PYPAPI library to count average flops for model inference. - Note: It only works if the model is being run on cpu""" - logger.info("Starting flop counter") - high.start_counters([events.PAPI_DP_OPS]) - for i, sample in enumerate(dataset): - for _r in range(repeat): - self.forward(sample) - if i % 100 == 0: - logger.info(f"Counted flops for {i}/{len(dataset)} samples") - flops = high.stop_counters() - flops = round(flops[0] / (repeat * len(dataset))) - return flops - - def max_memory(self, dataset, repeat): - """Compute average max memory consumed by model inference. Units are MiB""" - logger.info("Starting memory benchmarking") - total_memory = 0 - for i, sample in enumerate(dataset): - for _r in range(repeat): - total_memory += max(memory_usage((self.forward, (sample,), {}))) - if i % 100 == 0: - logger.info(f"Benchmarked memory for {i}/{len(dataset)} samples") - total_memory = total_memory / (repeat * len(dataset)) - return total_memory - - def gather_all_metrics(self, dataset, repeat): - run_time = self.benchmark_run_time(dataset, repeat) - max_memory = self.max_memory(dataset, repeat) - flops = self.count_flops(dataset, repeat) - - return run_time, max_memory, flops - - def dump_final_speech_output( - self, dataset, output_dir, resample_fn, sample_rate, prefix=None - ): - - for i, sample in enumerate(dataset): - hypo = self.forward(sample)[0] - - def to_np(x): - return x.detach().cpu().numpy() - - try: - wave_preds = to_np(resample_fn(hypo["waveform"])) - sf.write( - f"{output_dir}/{prefix}_{i}_pred.wav", - wave_preds, - sample_rate, - ) - except Exception as e: - raise Exception( - f" Encountered {e} - Invalid waveform. Make sure the model outputs a waveform" - ) - - -class Processing(BenchmarkingBase): - """Class similar to fairseq_cli/generate.py. Supports ASR, MT and ST model inference""" - - def __init__(self, args): - super().__init__() - self.use_cuda = not getattr(args, "cpu", False) - self.setUp(args) - self.training = False - self.s2x_task = self.task - - def setUp(self, cfg): - if isinstance(cfg, Namespace): - cfg = convert_namespace_to_omegaconf(cfg) - - self.task = tasks.setup_task(cfg.task) - self.tgt_dict = self.task.target_dictionary - - # Load ensemble - logger.info("loading model(s) from {}".format(cfg.common_eval.path)) - models, _ = checkpoint_utils.load_model_ensemble( - utils.split_paths(cfg.common_eval.path), - arg_overrides={}, - task=self.task, - suffix=cfg.checkpoint.checkpoint_suffix, - strict=False, - num_shards=cfg.checkpoint.checkpoint_shard_count, - ) - if len(models) > 1: - raise Exception("Currently loading multiple models is not supported") - self.model = models[0] - - # Optimize model for generation - if cfg.common.fp16: - self.model.half() - if self.use_cuda: - self.model.cuda() - self.model.prepare_for_inference_(cfg) - - self.generator = self.task.build_generator( - [self.model], - cfg.generation, - extra_gen_cls_kwargs={}, - ) - # Handle tokenization and BPE - self.tokenizer = self.task.build_tokenizer(cfg.tokenizer) - self.bpe = self.task.build_bpe(cfg.bpe) - self.remove_bpe = cfg.common_eval.post_process - - def encode_source(self, src): - """Method to generate source tokens from a string""" - if self.tokenizer is not None: - src = self.tokenizer.encode(src) - if self.bpe is not None: - src = self.bpe.encode(src) - src_tokens = self.task.source_dictionary.encode_line(src).long() - src_lens = src_tokens.size(0) - return { - "net_input": { - "src_tokens": src_tokens.view(1, src_lens), - "src_lengths": torch.tensor([src_lens]), - } - } - - def decode_target(self, hypos): - """Method to decode target string from tokens""" - hypo_str = self.tgt_dict.string( - hypos[0][0]["tokens"].int().cpu(), - self.remove_bpe, - get_symbols_to_strip_from_output(self.generator), - ) - if self.bpe is not None: - hypo_str = self.bpe.decode(hypo_str) - if self.tokenizer is not None: - hypo_str = self.tokenizer.decode(hypo_str) - return hypo_str - - def forward(self, sample): - hypos = self.task.inference_step( - self.generator, - [self.model], - sample, - prefix_tokens=None, - constraints=None, - ) - return hypos - - -class GenerateWaveformFromCode(BenchmarkingBase): - """Class to support waveform generation from code. Currently, vocoder only supports single speaker""" - - def __init__(self, args): - super().__init__() - with open(args.vocoder_cfg) as f: - vocoder_cfg = json.load(f) - self.dur_prediction = args.dur_prediction - self.vocoder = CodeHiFiGANVocoder(args.vocoder, vocoder_cfg) - - def format_units(self, input): - code = torch.LongTensor(list(map(int, input.strip().split()))).view(1, -1) - return {"code": code} - - def generate_vocoder_input(self, dataset): - return [self.format_units(sample) for sample in dataset] - - def forward(self, sample): - return [{"waveform": self.vocoder(sample, self.dur_prediction)}] - - -class HubertUnitExtractor(BenchmarkingBase): - def __init__(self, args): - self.feature_reader = HubertFeatureReader( - args.hubert_ckpt_path, args.hubert_layer - ) - self.kmeans = ApplyKmeans(args.hubert_km_path) - - def forward(self, sample): - with torch.no_grad(): - feat = [] - for start in range(0, sample.size(1), self.feature_reader.max_chunk): - x_chunk = sample[:, start : start + self.max_chunk] - feat_chunk, _ = self.feature_reader.model.extract_features( - source=x_chunk, - padding_mask=None, - mask=False, - output_layer=self.layer, - ) - feat.append(feat_chunk) - torch.cat(feat, 1).squeeze(0) - return self.kmeans(feat).tolist() - - -class SpeechGeneration(BenchmarkingBase): - """Class similar to examples/text_to_speech/generate_waveform.py. - Supports models with speech generation as end goal (TTS, Direct S2ST models etc)""" - - def __init__(self, args): - super().__init__() - self.use_cuda = not getattr(args, "cpu", False) - self.setUp(args) - self.s2x_task = self.task - - def setUp(self, args): - if args.task == "speech_to_speech": - args.normalize_waveform = False - self.task = tasks.setup_task(args) - self.pre_tokenizer = self.task.build_tokenizer(args) - self.bpe_tokenizer = self.task.build_bpe(args) - try: - self.src_dict = self.task.src_dict - except Exception: - self.src_dict = None - ensemble, saved_cfg, task = checkpoint_utils.load_model_ensemble_and_task( - [args.path], - arg_overrides=ast.literal_eval(args.model_overrides), - task=self.task, - strict=False, - ) - self.model = ensemble[0] - if self.use_cuda: - self.model.cuda() - # criterion.cuda() - self.model.eval() - self.generator = self.task.build_generator( - [self.model], - args, - ) - - def processTextInput(self, text): - """Generate source tokens from text input""" - if self.pre_tokenizer is not None: - text = self.pre_tokenizer.encode(text) - if self.bpe_tokenizer is not None: - text = self.bpe_tokenizer.encode(text) - target = self.src_dict.encode_line( - text, add_if_not_exist=False, append_eos=True - ).long() - target = fairseq_data_utils.collate_tokens( - [target], - self.src_dict.pad(), - self.src_dict.eos(), - left_pad=False, - move_eos_to_beginning=False, - ) - src_lengths = torch.tensor([target.size(1)], dtype=torch.long) - prev_output_tokens = None - sample = { - "net_input": { - "src_tokens": target, - "src_lengths": src_lengths, - "prev_output_tokens": prev_output_tokens, - } - } - sample = utils.move_to_cuda(sample) if self.use_cuda else sample - return sample - - def forward(self, sample): - sample["speaker"] = None - output = self.generator.generate(self.model, sample) # , has_targ=False - return output - - -class S2UT(BenchmarkingBase): - """Class to support S2UT models. Also supports generating waveforms from the units predicted""" - - def __init__(self, s2u_args, vocoder_args=None): - super().__init__() - self.s2u = Processing(s2u_args) - self.vocoder = None - if vocoder_args: - self.vocoder = GenerateWaveformFromCode(vocoder_args) - self.vocoder_input = None - - def forward(self, sample): - s2u_hypos = self.s2u(sample) - s2u_output = self.s2u.decode_target(s2u_hypos) - if not self.vocoder: - return s2u_output - units = self.vocoder.format_units(s2u_output) - vocoder_output = self.vocoder(units) - return vocoder_output - - def generate_s2u_outputs(self, dataset): - return [self.s2u.decode_target(self.s2u(sample)) for sample in dataset] - - def compute_metrics(self, metric_type, dataset, repeat=None): - """Generic function to compute metrics ignoring the io processing time""" - if self.vocoder and not self.vocoder_input: - self.s2u_output = self.generate_s2u_outputs(dataset) - self.vocoder_input = self.vocoder.generate_vocoder_input(self.s2u_output) - - s2u_metrics = getattr(self.s2u, metric_type)( - dataset, - repeat, - ) - vocoder_metrics = 0 - if self.vocoder: - vocoder_metrics = getattr(self.vocoder, metric_type)( - self.vocoder_input, - repeat, - ) - print( - f"metric_type = {metric_type} s2u_metrics = {s2u_metrics} \t vocoder_metrics = {vocoder_metrics}" - ) - if metric_type == "max_memory": - return max(s2u_metrics, vocoder_metrics) - else: - return s2u_metrics + vocoder_metrics - - def benchmark_run_time(self, dataset, repeat): - return self.compute_metrics("benchmark_run_time", dataset, repeat) - - def count_flops(self, dataset, repeat): - return self.compute_metrics("count_flops", dataset, repeat) - - def max_memory(self, dataset, repeat): - return self.compute_metrics("max_memory", dataset, repeat) - - -class Cascaded2StageS2ST(BenchmarkingBase): - """ST + TTS""" - - def __init__(self, s2t_args, tts_args): - super().__init__() - self.s2t = Processing(s2t_args) - self.s2x_task = self.s2t.task - self.tts = SpeechGeneration(tts_args) if tts_args else None - self.training = False - self.tts_inputs = None - - def forward(self, sample): - if not self.tts: - raise Exception( - "Forward function is not callable without tts. Reinitialize the class with tts_args" - ) - s2t_hypos = self.s2t(sample) - s2t_output = self.s2t.decode_target(s2t_hypos) - tts_input = self.tts.processTextInput(s2t_output) - tts_output = self.tts(tts_input) - return tts_output - - def generate_s2t_outputs(self, dataset): - """Process dataset and generate s2t outputs""" - return [self.s2t.decode_target(self.s2t(sample)) for sample in dataset] - - def generate_tts_inputs(self, dataset): - """Process dataset and generate tts inputs""" - return [self.tts.processTextInput(sample) for sample in dataset] - - def compute_metrics(self, metric_type, dataset, repeat=None): - """Generic function to compute metrics ignoring the io processing time""" - if not self.tts_inputs: - s2t_outputs = self.generate_s2t_outputs(dataset) - self.tts_inputs = self.generate_tts_inputs(s2t_outputs) - - s2t_metrics = getattr(self.s2t, metric_type)( - dataset, - repeat, - ) - - tts_metrics = getattr(self.tts, metric_type)( - self.tts_inputs, - repeat, - ) - print( - f"metric_type = {metric_type} s2t_metrics = {s2t_metrics} \t tts_metrics = {tts_metrics}" - ) - if metric_type == "max_memory": - return max(s2t_metrics, tts_metrics) - else: - return s2t_metrics + tts_metrics - - def benchmark_run_time(self, dataset, repeat): - return self.compute_metrics("benchmark_run_time", dataset, repeat) - - def count_flops(self, dataset, repeat): - return self.compute_metrics("count_flops", dataset, repeat) - - def max_memory(self, dataset, repeat): - return self.compute_metrics("max_memory", dataset, repeat) - - -class Cascaded3StageS2ST(Cascaded2StageS2ST): - """ASR + MT + TTS""" - - def __init__(self, s2t_args, tts_args, mt_args): - super().__init__(s2t_args, tts_args) - self.mt = Processing(mt_args) - self.mt_inputs = [] - - def forward(self, sample): - s2t_hypos = self.s2t(sample) - s2t_output = self.s2t.decode_target(s2t_hypos) - mt_input = self.mt.encode_source(s2t_output) - mt_hypos = self.mt(mt_input) - mt_output = self.mt.decode_target(mt_hypos) - tts_input = self.tts.processTextInput(mt_output) - tts_output = self.tts(tts_input) - return tts_output - - def generate_mt_inputs(self, dataset): - """Process dataset to generate mt model inputs""" - return [self.mt.encode_source(sample) for sample in dataset] - - def generate_mt_outputs(self, dataset): - """Process dataset to generate mt model outputs""" - return [self.mt.decode_target(self.mt(sample)) for sample in dataset] - - def compute_metrics(self, metric_type, dataset, repeat=None): - """Generic function to compute metrics ignoring the io processing time""" - if not self.tts_inputs: - s2t_outputs = self.generate_s2t_outputs(dataset) - self.mt_inputs = self.generate_mt_inputs(s2t_outputs) - mt_outputs = self.generate_mt_outputs(self.mt_inputs) - self.tts_inputs = self.generate_tts_inputs(mt_outputs) - - s2t_metrics = getattr(self.s2t, metric_type)( - dataset, - repeat, - ) - mt_metrics = getattr(self.mt, metric_type)(self.mt_inputs, repeat) - tts_metrics = getattr(self.tts, metric_type)( - self.tts_inputs, - repeat, - ) - print( - f"metric_type = {metric_type} s2t_metrics = {s2t_metrics} \t mt_metrics = {mt_metrics} \t tts_metrics = {tts_metrics}" - ) - if metric_type == "max_memory": - return max(s2t_metrics, mt_metrics, tts_metrics) - else: - return s2t_metrics + mt_metrics + tts_metrics diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/benchmarking/data_utils.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/benchmarking/data_utils.py deleted file mode 100644 index dec7a09a087ad89a97d4cce246e364189fce491f..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/benchmarking/data_utils.py +++ /dev/null @@ -1,263 +0,0 @@ -import logging -import os -import random - -import numpy as np -import soundfile as sf -import torch -from fairseq import options, tasks -from fairseq.data.audio.audio_utils import ( - get_waveform, - parse_path, -) - -logging.basicConfig() -logging.root.setLevel(logging.INFO) -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - -random.seed(1) -np.random.seed(1) -random_number_generator = np.random.RandomState(30) - - -def generate_random_data_sample(T, B=1, D=80): - """Generate random data sample given the T, B, D values""" - net_input = { - "src_tokens": torch.tensor(random_number_generator.randn(B, T, D)).float(), - "src_lengths": torch.tensor([T]), - } - return {"net_input": net_input} - - -def generate_random_dataset(T_range_min, T_range_max, B=1, D=80, dataset_size=100): - """Generate random dataset with T values within a given range, B, D""" - T_values = [random.randint(T_range_min, T_range_max) for i in range(dataset_size)] - dataset = [] - for t in T_values: - dataset.append(generate_random_data_sample(t, B, D)) - return dataset, sum(T_values) / dataset_size - - -def load_dataset_npy(file_name, dataset_size=None): - """Load dataset from a .npy file.""" - data = np.load(file_name, allow_pickle=True) - if dataset_size: - data = data[:dataset_size] - return data - - -def load_dataset_raw_to_waveforms( - file_name, - dataset_size=None, - need_waveform=True, - sample_rate=16000, - read_using_soundfile=False, -): - """Load raw dataset from w2v tsv file. Optionally get waveforms""" - data = [] - with open(file_name, "r") as fp: - lines = fp.readlines() - data = [ - os.path.join(lines[0].strip(), line.strip().split("\t")[0]) - for line in lines[1:] - ] - - if dataset_size: - data = data[:dataset_size] - - if not need_waveform: - return data - - features = [] - if read_using_soundfile: - for _i, d in enumerate(data): - wav = sf.read(d)[0] - if wav.ndim == 2: - wav = wav.mean(-1) - features.append(torch.from_numpy(wav).float().view(1, -1)) - else: - for i, d in enumerate(data): - _path, slice_ptr = parse_path(d) - if len(slice_ptr) == 0: - feat = get_waveform( - _path, always_2d=True, output_sample_rate=sample_rate - )[0] - features.append( - { - "id": i, - "net_input": { - "src_tokens": torch.tensor(feat), - "src_lengths": torch.tensor([feat.shape[1]]), - }, - } - ) - else: - raise Exception("Currently unsupported data format") - return features - - -def load_dataset_task( - args, - batch_size=1, - limit_size=None, - ref_dataset=None, -): - """Loads dataset based on args by creating a task""" - if not args.data or not args.subset or not args.task: - raise Exception( - "Please provide necessary arguments to load the dataset - data, subset and task" - ) - task = tasks.setup_task(args) - - task.load_dataset(args.subset) - if not limit_size: - limit_size = len(task.dataset(args.subset)) - - iter = task.get_batch_iterator( - dataset=task.dataset(args.subset), max_sentences=batch_size - ).next_epoch_itr(shuffle=False) - dataset = [] - for i, sample in enumerate(iter): - sample = { - "id": task.datasets[args.subset].ids[sample["id"].item()], - "net_input": { - "src_tokens": sample["net_input"]["src_tokens"], - "src_lengths": sample["net_input"]["src_lengths"], - }, - } - dataset.append(sample) - if i == limit_size - 1: - break - - if ref_dataset: - try: - ids = get_ids_from_dataset(ref_dataset) - except Exception as e: - raise Exception(f"{e} - Cannot extract ids from reference dataset") - - filtered_dataset = [] - for sample in dataset: - if ( - sample["id"] in ids - or sample["id"][5:] in ids - or f"dev_{sample['id']}" in ids - ): - filtered_dataset.append(sample) - dataset = filtered_dataset - - max_len, min_len, avg_len = get_dataset_stats(dataset) - print( - f"{args.subset} dataset stats : num_samples={len(dataset)} max_len = {max_len} min_len = {min_len} avg_len = {avg_len}" - ) - - return dataset - - -def randomly_sample_subset(dataset, size=500): - """Randomly sample subset from a dataset""" - random_indices = [random.randint(0, len(dataset) - 1) for i in range(size)] - return [dataset[i] for i in random_indices] - - -def get_short_data_subset(dataset, size=500): - """Get a subset of desired size by sorting based on src_lengths""" - return sort_dataset(dataset)[:size] - - -def get_long_data_subset(dataset, size=500): - """Get a subset of desired size by sorting based on src_lengths descending""" - return sort_dataset(dataset, reverse=True)[:size] - - -def sort_dataset(dataset, reverse=False): - return sorted( - dataset, key=lambda x: x["net_input"]["src_lengths"].item(), reverse=reverse - ) - - -def save_dataset_npy(dataset, file_name): - """Save a dataset as .npy file""" - np.save(file_name, dataset) - - -def get_dataset_stats(dataset): - """Get stats about dataset based on src_lengths of samples""" - max_len = 0 - min_len = 100000 - avg_len = 0 - for d in dataset: - max_len = max(max_len, d["net_input"]["src_lengths"].item()) - min_len = min(min_len, d["net_input"]["src_lengths"].item()) - avg_len += d["net_input"]["src_lengths"].item() - - return max_len, min_len, avg_len / len(dataset) - - -def make_parser(): - """ - Additional args: - 1. Provide the dataset dir path using --data. - 2. Loading the dataset doesn't require config, provide --config-yaml to apply additional feature transforms - """ - parser = options.get_speech_generation_parser() - parser.add_argument( - "--subset", - default=None, - type=str, - required=True, - help="Subset to use for dataset generation", - ) - parser.add_argument( - "--dataset-save-dir", - default=None, - type=str, - required=False, - help="Dir path in which the datasets are to be saved", - ) - parser.add_argument( - "--ref-dataset", - default=None, - type=str, - required=False, - help="If provided, the ids in the reference dataset will be used to filter the new dataset generated.", - ) - parser.add_argument("--dataset-save-token", default="", type=str, required=False) - - options.add_generation_args(parser) - return parser - - -def get_ids_from_dataset(dataset): - return {sample["id"]: 1 for sample in dataset} - - -def cli_main(): - parser = make_parser() - args = options.parse_args_and_arch(parser) - dataset = load_dataset_task(args) - - random_dataset = randomly_sample_subset(dataset) - short_dataset = get_short_data_subset(dataset) - long_dataset = get_long_data_subset(dataset) - - if args.dataset_save_token: - args.dataset_save_token = f"_{args.dataset_save_token}_" - - if args.dataset_save_dir: - save_dataset_npy( - random_dataset, - f"{args.dataset_save_dir}/random_dataset{args.dataset_save_token}w_ids.npy", - ) - save_dataset_npy( - short_dataset, - f"{args.dataset_save_dir}/short_dataset{args.dataset_save_token}w_ids.npy", - ) - save_dataset_npy( - long_dataset, - f"{args.dataset_save_dir}/long_dataset{args.dataset_save_token}w_ids.npy", - ) - - -if __name__ == "__main__": - cli_main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/benchmarking/get_metrics.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/benchmarking/get_metrics.py deleted file mode 100644 index e206df1a0c0f03ff1c3bf7450c06aa8a177165a2..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/benchmarking/get_metrics.py +++ /dev/null @@ -1,162 +0,0 @@ -import copy -import logging -from argparse import Namespace - -import torch -import yaml -from examples.speech_to_speech.benchmarking.core import ( - S2UT, - Cascaded2StageS2ST, - Cascaded3StageS2ST, - Processing, - SpeechGeneration, -) -from examples.speech_to_speech.benchmarking.data_utils import ( - load_dataset_npy, - load_dataset_raw_to_waveforms, -) -from fairseq import options - -logging.basicConfig() -logging.root.setLevel(logging.INFO) -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - -torch.manual_seed(1) -torch.set_deterministic(True) - - -def make_parser(): - """Note: As the names indicate use s2x_args(ex:ST, ASR etc) for models with speech input, - x2s_args for models with speech output(ex:TTS) and mt_args for translation models (ex: mt, T2U etc). - For direct S2ST models, use x2s_args to provide model details. - """ - parser = options.get_speech_generation_parser() - parser.add_argument("--target-is-code", action="store_true", default=False) - parser.add_argument("--config", type=str) - parser.add_argument( - "--model-type", - default="S2U", - choices=["S2S", "TTS", "S2UT", "MT", "S2T", "2StageS2ST", "3StageS2ST"], - help="Choose one of the models. For model inference implementation, refer to core.py", - ) - parser.add_argument( - "--dataset-path", - type=str, - help="""File to load dataset from. Assumes dataset is a list of samples. - Each sample is a dict of format {'net_input':{'src_tokens':torch.tenor(),'src_lengths':torch.tensor()}}""", - ) - parser.add_argument( - "--dataset-type", - type=str, - default="npy", - choices=["npy", "raw"], - help="""Type of input dataset file""", - ) - parser.add_argument( - "--read-using-sf", - type=str, - default=False, - help="""If sound file should be used to read the raw dataset""", - ) - parser.add_argument( - "--dataset-size", - default=None, - type=int, - help="Dataset size to use for benchmarking", - ) - parser.add_argument( - "--dump-speech-waveforms-dir", - default=None, - type=str, - help="Directory to dump the speech waveforms computed on the dataset.", - ) - parser.add_argument( - "--dump-waveform-file-prefix", - default="", - type=str, - help="File name prefix for the saved speech waveforms", - ) - parser.add_argument( - "--feat-dim", default=80, type=int, help="Input feature dimension" - ) - parser.add_argument( - "--target-sr", - default=16000, - type=int, - help="Target sample rate for dumping waveforms", - ) - - options.add_generation_args(parser) - options.get_interactive_generation_parser(parser) - return parser - - -def cli_main(): - parser = make_parser() - args = options.parse_args_and_arch(parser) - - with open( - args.config, - "r", - ) as f: - config = yaml.load(f, Loader=yaml.FullLoader) - dict_args = vars(args) - dict_args.update(config["general"]) - args = Namespace(**dict_args) - - i = 1 - stage_args = [] - while i <= 3: - var = f"stage{i}" - tmp_args = copy.deepcopy(dict_args) - if var in config: - tmp_args.update(config[var]) - stage_args.append(Namespace(**tmp_args)) - i += 1 - else: - break - - if args.model_type == "S2S" or args.model_type == "TTS": - model = SpeechGeneration(stage_args[0]) - elif args.model_type == "S2UT": - model = S2UT(stage_args[0], stage_args[1] if len(stage_args) > 1 else None) - elif args.model_type == "MT" or args.model_type == "S2T": - model = Processing(stage_args[0]) - elif args.model_type == "2StageS2ST": - model = Cascaded2StageS2ST(stage_args[0], stage_args[1]) - elif args.model_type == "3StageS2ST": - model = Cascaded3StageS2ST(stage_args[0], stage_args[2], stage_args[1]) - else: - raise Exception(f"Currently unsupported model type {args.model_type}") - - print(f"Evaluating on dataset - {args.dataset_path}\n") - - if args.dataset_type == "npy": - dataset = load_dataset_npy(args.dataset_path, dataset_size=args.dataset_size) - elif args.dataset_type == "raw": - dataset = load_dataset_raw_to_waveforms( - args.dataset_path, - dataset_size=args.dataset_size, - read_using_soundfile=args.read_using_sf, - ) - else: - raise Exception(f"Invalid dataset type {args.dataset_type}") - - model.warm_up(sample=dataset[0], repeat=2) - - run_time, memory, flops = model.gather_all_metrics(dataset, repeat=1) - print(f"run_time = {run_time}sec \tmemory = {memory}MiB \tflops = {flops}") - - if args.dump_speech_waveforms_dir: - model.dump_final_speech_output( - dataset, - args.dump_speech_waveforms_dir, - lambda x: x, - args.target_sr, - prefix=args.dump_waveform_file_prefix, - ) - - -if __name__ == "__main__": - cli_main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/docs/direct_s2st_discrete_units.md b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/docs/direct_s2st_discrete_units.md deleted file mode 100644 index 0c63ffee1cc4eff67cb3b761e4771dbf33f36e1a..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/docs/direct_s2st_discrete_units.md +++ /dev/null @@ -1,181 +0,0 @@ -# Direct speech-to-speech translation with discrete units - -We provide the implementation for speech-to-unit translation (S2UT) proposed in "[Direct speech-to-speech translation with discrete units (Lee et al. 2021)](https://arxiv.org/abs/2107.05604)" and also the transformer-based implementation of the speech-to-spectrogram translation (S2SPECT, or transformer-based [Translatotron](https://arxiv.org/abs/1904.06037)) baseline in the paper. - -## Pretrained Models - -### Unit-based HiFi-GAN Vocoder -Unit config | Unit size | Vocoder dataset | Model -|---|---|---|--- -[HuBERT Base, Librispeech](https://github.com/fairinternal/fairseq-py/tree/main/examples/hubert), layer 6 | 100 | [LJSpeech](https://keithito.com/LJ-Speech-Dataset/) | [ckpt](https://dl.fbaipublicfiles.com/fairseq/speech_to_speech/vocoder/code_hifigan/hubert_base_100_lj/g_00500000), [config](https://dl.fbaipublicfiles.com/fairseq/speech_to_speech/vocoder/code_hifigan/hubert_base_100_lj/config.json) - - -## Data preparation -### Target speech -0. (optional) To prepare S2S data from a speech-to-text translation (ST) dataset, see [fairseq-S^2](https://github.com/pytorch/fairseq/tree/main/examples/speech_synthesis) for pre-trained TTS models and instructions on how to train and decode TTS models. -1. Prepare two folders, `$SRC_AUDIO` and `$TGT_AUDIO`, with `${SPLIT}/${SAMPLE_ID}.wav` for source and target speech under each folder, separately. Note that for S2UT experiments, target audio sampling rate should be in 16,000 Hz, and for S2SPECT experiments, target audio sampling rate is recommended to be in 22,050 Hz. -2. To prepare target discrete units for S2UT model training, see [Generative Spoken Language Modeling (speech2unit)](https://github.com/pytorch/fairseq/tree/main/examples/textless_nlp/gslm/speech2unit) for pre-trained k-means models, checkpoints, and instructions on how to decode units from speech. Set the output target unit files (`--out_quantized_file_path`) as `${TGT_AUDIO}/${SPLIT}.txt`. In [Lee et al. 2021](https://arxiv.org/abs/2107.05604), we use 100 units from the sixth layer (`--layer 6`) of the HuBERT Base model. - -### Formatting data -**Speech-to-speech data** - -_S2UT_ - * Set `--reduce-unit` for training S2UT _reduced_ model - * Pre-trained vocoder and config (`$VOCODER_CKPT`, `$VOCODER_CFG`) can be downloaded from the **Pretrained Models** section. They are not required if `--eval-inference` is not going to be set during model training. -``` -# $SPLIT1, $SPLIT2, etc. are split names such as train, dev, test, etc. - -python examples/speech_to_speech/preprocessing/prep_s2ut_data.py \ - --source-dir $SRC_AUDIO --target-dir $TGT_AUDIO --data-split $SPLIT1 $SPLIT2 \ - --output-root $DATA_ROOT --reduce-unit \ - --vocoder-checkpoint $VOCODER_CKPT --vocoder-cfg $VOCODER_CFG -``` - -_S2SPECT_ -``` -# $SPLIT1, $SPLIT2, etc. are split names such as train, dev, test, etc. - -python examples/speech_to_speech/preprocessing/prep_s2spect_data.py \ - --source-dir $SRC_AUDIO --target-dir $TGT_AUDIO --data-split $SPLIT1 $SPLIT2 \ - --output-root $DATA_ROOT -``` - -**Multitask data** - * For each multitask `$TASK_NAME`, prepare `${DATA_ROOT}/${TASK_NAME}/${SPLIT}.tsv` files for each split following the format below: (Two tab separated columns. The sample_ids should match with the sample_ids for the speech-to-speech data in `${DATA_ROOT}/${SPLIT}.tsv`.) -``` -id tgt_text -sample_id_0 token1 token2 token3 ... -sample_id_1 token1 token2 token3 ... -... -``` - * For each multitask `$TASK_NAME`, prepare `${DATA_ROOT}/${TASK_NAME}/dict.txt`, a dictionary in fairseq format with all tokens for the targets for `$TASK_NAME`. - * Create `config_multitask.yaml`. Below is an example of the config used for S2UT _reduced_ with Fisher experiments including two encoder multitasks (`source_letter`, `target_letter`) and one decoder CTC task (`decoder_target_ctc`). -``` -source_letter: # $TASK_NAME - decoder_type: transformer - dict: ${DATA_ROOT}/source_letter/dict.txt - data: ${DATA_ROOT}/source_letter - encoder_layer: 6 - loss_weight: 8.0 -target_letter: - decoder_type: transformer - dict: ${DATA_ROOT}/target_letter/dict.txt - data: ${DATA_ROOT}/target_letter - encoder_layer: 8 - loss_weight: 8.0 -decoder_target_ctc: - decoder_type: ctc - dict: ${DATA_ROOT}/decoder_target_ctc/dict.txt - data: ${DATA_ROOT}/decoder_target_ctc - decoder_layer: 3 - loss_weight: 1.6 -``` - - -## Training - -**Speech-to-unit translation (S2UT)** - -Here's an example for training Fisher S2UT models with 100 discrete units as target: -``` -fairseq-train $DATA_ROOT \ - --config-yaml config.yaml --multitask-config-yaml config_multitask.yaml \ - --task speech_to_speech --target-is-code --target-code-size 100 --vocoder code_hifigan \ - --criterion speech_to_unit --label-smoothing 0.2 \ - --arch s2ut_transformer_fisher --share-decoder-input-output-embed \ - --dropout 0.1 --attention-dropout 0.1 --relu-dropout 0.1 \ - --train-subset train --valid-subset dev \ - --save-dir ${MODEL_DIR} \ - --lr 0.0005 --lr-scheduler inverse_sqrt --warmup-init-lr 1e-7 --warmup-updates 10000 \ - --optimizer adam --adam-betas "(0.9,0.98)" --clip-norm 10.0 \ - --max-update 400000 --max-tokens 20000 --max-target-positions 3000 --update-freq 4 \ - --seed 1 --fp16 --num-workers 8 -``` -* Adjust `--update-freq` accordingly for different #GPUs. In the above we set `--update-freq 4` to simulate training with 4 GPUs. -* Set `--n-frames-per-step 5` to train an S2UT _stacked_ system with reduction ratio r=5. (Use `$DATA_ROOT` prepared without `--reduce-unit`.) -* (optional) one can turn on tracking MCD loss during training for checkpoint selection by setting `--eval-inference --eval-args '{"beam": 1, "max_len_a": 1}' --best-checkpoint-metric mcd_loss`. It is recommended to sample a smaller subset as the validation set as MCD loss computation is time-consuming. - -**Speech-to-spectrogram translation (S2SPECT)** - -Here's an example for training Fisher S2SPECT models with reduction ratio r=5: -``` -fairseq-train $DATA_ROOT \ - --config-yaml config.yaml --multitask-config-yaml config_multitask.yaml \ - --task speech_to_speech --n-frames-per-step 5 \ - --criterion speech_to_spectrogram \ - --arch s2spect_transformer_fisher --decoder-normalize-before \ - --dropout 0.1 --attention-dropout 0.1 --relu-dropout 0.1 \ - --train-subset train --valid-subset dev \ - --save-dir ${MODEL_DIR} \ - --eval-inference --best-checkpoint-metric mcd_loss \ - --lr 0.0005 --lr-scheduler inverse_sqrt --warmup-init-lr 1e-7 --warmup-updates 10000 \ - --optimizer adam --adam-betas "(0.9,0.98)" --clip-norm 10.0 --weight-decay 1e-6 \ - --max-update 400000 --max-tokens 80000 --max-tokens-valid 30000 --required-batch-size-multiple 1 \ - --max-target-positions 3000 --update-freq 16 \ - --seed 1 --fp16 --num-workers 8 -``` -* Adjust `--update-freq` accordingly for different #GPUs. In the above we set `--update-freq 16` to simulate training with 16 GPUs. -* We recommend turning on MCD loss during training for the best checkpoint selection. - -**Unit-based HiFi-GAN vocoder** - -The vocoder is trained with the [speech-resynthesis repo](https://github.com/facebookresearch/speech-resynthesis). See [here](https://github.com/facebookresearch/speech-resynthesis/tree/main/examples/speech_to_speech_translation) for instructions on how to train the unit-based HiFi-GAN vocoder with duration prediction. The same vocoder can support waveform generation for both _reduced_ unit sequences (with `--dur-prediction` set during inference) and original unit sequences. - -## Inference - -**Speech-to-unit translation (S2UT)** - -1. Follow the same inference process as in [fairseq-S2T](https://github.com/pytorch/fairseq/tree/main/examples/speech_to_text) to generate unit sequences (`${RESULTS_PATH}/generate-${GEN_SUBSET}.txt`). -``` -fairseq-generate $DATA_ROOT \ - --config-yaml config.yaml --multitask-config-yaml config_multitask.yaml \ - --task speech_to_speech --target-is-code --target-code-size 100 --vocoder code_hifigan \ - --path $MODEL_DIR/checkpoint_best.pt --gen-subset $GEN_SUBSET \ - --max-tokens 50000 \ - --beam 10 --max-len-a 1 \ - --results-path ${RESULTS_PATH} -``` - * Set `--beam 1 --n-frames-per-step $r` for decoding with S2UT _stacked_ models. - -2. Convert unit sequences to waveform. -``` -grep "^D\-" ${RESULTS_PATH}/generate-${GEN_SUBSET}.txt | \ - sed 's/^D-//ig' | sort -nk1 | cut -f3 \ - > ${RESULTS_PATH}/generate-${GEN_SUBSET}.unit - -python examples/speech_to_speech/generate_waveform_from_code.py \ - --in-code-file ${RESULTS_PATH}/generate-${GEN_SUBSET}.unit \ - --vocoder $VOCODER_CKPT --vocoder-cfg $VOCODER_CFG \ - --results-path ${RESULTS_PATH} --dur-prediction -``` - * Set `--dur-prediction` for generating audio for S2UT _reduced_ models. - - -**Speech-to-spectrogram translation (S2SPECT)** - -Follow the same inference process as in [fairseq-S^2](https://github.com/pytorch/fairseq/tree/main/examples/speech_synthesis) to generate waveform. - -``` -# assume using a default Griffin-Lim vocoder - -python examples/speech_synthesis/generate_waveform.py $DATA_ROOT \ - --config-yaml config.yaml --multitask-config-yaml config_multitask.yaml \ - --task speech_to_speech --n-frames-per-step 5 \ - --path $MODEL_DIR/checkpoint_best.pt --gen-subset $GEN_SUBSET \ - --max-tokens 50000 \ - --results-path ${RESULTS_PATH} --dump-waveforms --output-sample-rate 16000 -``` - -In addition to using the default Griffin-Lim vocoder, one can also finetune a HiFi-GAN vocoder for the S2SPECT model by following the instructions in the [HiFi-GAN repo](https://github.com/jik876/hifi-gan). - -**Multitask decoding** - -Coming soon. - -## Evaluation - -To evaluate speech translation output, we first apply ASR on the speech output and then compute BLEU score betweent the ASR decoded text and the references using sacreBLEU. - -**En** -* ASR: We use the "[Wav2Vec 2.0 Large (LV-60) + Self Training / 960 hours / Libri-Light + Librispeech](https://dl.fbaipublicfiles.com/fairseq/wav2vec/wav2vec_vox_960h_pl.pt)" En ASR model open-sourced by the [wav2vec](https://github.com/pytorch/fairseq/tree/main/examples/wav2vec) project. See [instructions](https://github.com/pytorch/fairseq/tree/main/examples/wav2vec#evaluating-a-ctc-model) on how to run inference with a wav2vec-based ASR model. The model is also available on [Hugging Face](https://huggingface.co/facebook/wav2vec2-large-960h-lv60-self). -* Text normalization: We use the text cleaner at [https://github.com/keithito/tacotron](https://github.com/keithito/tacotron) for pre-processing reference English text for ASR BLEU evaluation. diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/docs/enhanced_direct_s2st_discrete_units.md b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/docs/enhanced_direct_s2st_discrete_units.md deleted file mode 100644 index 55bf22100e3c854eea7dca3011e2ad459fe1f76a..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/docs/enhanced_direct_s2st_discrete_units.md +++ /dev/null @@ -1,108 +0,0 @@ -# Speech to speech translation (S2ST) - -We provide the implementation for speech-to-unit translation (S2UT) proposed in [Enhanced Direct Speech-to-Speech Translation Using Self-supervised Pre-training and Data Augmentation (Popuri et al. 2022)](https://arxiv.org/abs/2204.02967) and the various pretrained models used. - -## Pretrained Models - -### Unit extraction - -We used the multilingual HuBERT model open sourced in [Textless S2ST with Real Data](textless_s2st_real_data.md) - -### Wav2vec 2.0 - -Language | Block type | Model size | Dataset | Model | ---- | --- | --- | --- | --- | -Es | Transformer | BASE | Voxpopuli | [ckpt](https://dl.fbaipublicfiles.com/fairseq/speech_to_speech/s2st_finetuning/w2v2/es/transformer_B.pt) | -Es | Transformer | LARGE | Voxpopuli | [ckpt](https://dl.fbaipublicfiles.com/fairseq/speech_to_speech/s2st_finetuning/w2v2/es/transformer_L.pt) | -Es | Conformer | LARGE | Voxpopuli | [ckpt](https://dl.fbaipublicfiles.com/fairseq/speech_to_speech/s2st_finetuning/w2v2/es/conformer_L.pt) | -En | Transformer | BASE | Librilight| [ckpt](https://dl.fbaipublicfiles.com/fairseq/speech_to_speech/s2st_finetuning/w2v2/en/transformer_B.pt) | -En | Conformer | LARGE | Librilight | [ckpt](https://dl.fbaipublicfiles.com/fairseq/speech_to_speech/s2st_finetuning/w2v2/en/conformer_L.pt) | - -### Unit mBART - -Unit size | Dataset | Unit config | Model | ---- | --- | --- | --- | -1000 | [Voxpopuli](https://aclanthology.org/2021.acl-long.80) En, Es unlabelled speech | [mbart_large](https://github.com/pytorch/fairseq/blob/f591cc94caa85098ccf125a4782f91125b6a086d/fairseq/models/bart/model.py#L368) |[ckpt](https://dl.fbaipublicfiles.com/fairseq/speech_to_speech/s2st_finetuning/unit_mBART/checkpoint.pt) | - -## Data preparation - -1. To prepare data for S2UT finetuning, follow the steps from [Direct S2ST with Discrete Units](./direct_s2st_discrete_units.md) and format the data in the _S2UT_ format. Note that we use 1000 units from the eleventh layer (`--layer 11`) of the multilingual hubert model linked above instead -2. Run - -``` -var="id\taudio\tn_frames\ttgt_text\ttgt_n_frames" -sed -i "1s/.*/$var/" ${SPLIT}.tsv -``` - -## Training - -**Speech-to-unit translation (S2UT)** - -Here's an example for finetuning S2UT models with 1000 discrete units as target. You can download the [config](https://dl.fbaipublicfiles.com/fairseq/speech_to_speech/s2st_finetuning/config.yaml) file and [vocabulary](https://dl.fbaipublicfiles.com/fairseq/speech_to_speech/s2st_finetuning/dict.txt) from here: - -``` -fairseq-train $DATA_ROOT \ - --config-yaml config.yaml \ - --task speech_to_text --arch xm_transformer\ - --criterion l --label-smoothing 0.2 \ - --share-decoder-input-output-embed --adaptor-n-layers 1 --normalize\ - --dropout 0.1 --attention-dropout 0.1 --relu-dropout 0.1 \ - --train-subset train --valid-subset dev \ - --load-pretrained-decoder-from ${unit_mBART} --w2v-path ${wav2vec2.0} \ - --mask-prob 0.3 --mask-channel-length 32 --mask-channel-prob 0.25\ - --save-dir ${MODEL_DIR} --checkpoint-activations --encoder-proj \ - --lr 0.0005 --dropout 0.1 --attention-dropout 0.1 --lr-scheduler inverse_sqrt\ - --warmup-init-lr 1e-7 --warmup-updates 10000 \ - --optimizer adam --adam-betas "(0.9,0.98)" --clip-norm 10.0 \ - --max-update 20000 --max-tokens 4000 --max-tokens-valid 4000 --max-source-positions 4000 \ - --max-target-positions 4000 --update-freq 120 \ - --seed 1 --fp16 --num-workers 1 -``` - -* Adjust `--update-freq` accordingly for different #GPUs. In the above we set `--update-freq 15` to simulate training with 120 GPUs. -* In the above setting we finetune the model end to end, corresponding to the full setup in the paper. -* To apply LNA-E partial finetuning, add `--finetune-w2v-params layer_norm,self_attn` -* For LNA-D partial finetuning add `--finetune-decoder-params encoder_attn,layer_norm,self_attn`. To optionally freeze the encoder by k updates, use `--freeze-finetune-updates ${K}` -* For LNA-E,D partial finetuning add both the above options. - -**Unit-based HiFi-GAN vocoder** - -We apply the open-sourced unit-based HiFi-GAN vocoders to convert the predicted unit sequences to waveform. They are open sourced in [Textless S2ST with Real Data](textless_s2st_real_data.md) - -## Inference - -**Speech-to-unit translation (S2UT)** - -1. Follow the same inference process as in [fairseq-S2T](https://github.com/pytorch/fairseq/tree/main/examples/speech_to_text) to generate unit sequences (`${RESULTS_PATH}/generate-${GEN_SUBSET}.txt`). - -``` -fairseq-generate $DATA_ROOT \ - --config-yaml config.yaml \ - --task speech_to_text \ - --path $MODEL_DIR/checkpoint_best.pt --gen-subset $GEN_SUBSET \ - --max-tokens 10000 --max-source-positions 10000 --max-target-positions 10000\ - --beam 10 --max-len-a 1 --max-len-b 200 \ - --results-path ${RESULTS_PATH} -``` - -2. Convert unit sequences to waveform. - -``` -grep "^D\-" ${RESULTS_PATH}/generate-${GEN_SUBSET}.txt | \ - sed 's/^D-//ig' | sort -nk1 | cut -f3 \ - > ${RESULTS_PATH}/generate-${GEN_SUBSET}.unit - -python examples/speech_to_speech/generate_waveform_from_code.py \ - --in-code-file ${RESULTS_PATH}/generate-${GEN_SUBSET}.unit \ - --vocoder $VOCODER_CKPT --vocoder-cfg $VOCODER_CFG \ - --results-path ${RESULTS_PATH} --dur-prediction -``` - -## Evaluation - -To evaluate speech translation output, we first apply ASR on the speech output and then compute BLEU score betweent the ASR decoded text and the references using sacreBLEU. - -* Text normalization: We use the text cleaner at [https://github.com/keithito/tacotron](https://github.com/keithito/tacotron) for pre-processing reference English text for ASR BLEU evaluation. The text cleaner used for Spanish text normalization will be updated here shortly. -* En ASR: We use the "[Wav2Vec 2.0 Large (LV-60) + Self Training / 960 hours / Libri-Light + Librispeech](https://dl.fbaipublicfiles.com/fairseq/wav2vec/wav2vec_vox_960h_pl.pt)" En ASR model open-sourced by the [wav2vec](https://github.com/pytorch/fairseq/tree/main/examples/wav2vec) project. The model is also available on [Hugging Face](https://huggingface.co/facebook/wav2vec2-large-960h-lv60-self). -* Es ASR: We use the [Wav2Vec2-Large-XLSR-53-Spanish](https://huggingface.co/facebook/wav2vec2-large-xlsr-53) finetuned on spanish Common Voice Es ASR model open-sourced by Jonatasgrosman() on [Hugging Face](https://huggingface.co/jonatasgrosman/wav2vec2-large-xlsr-53-spanish). -* See [instructions](https://github.com/pytorch/fairseq/tree/main/examples/wav2vec#evaluating-a-ctc-model) on how to run inference with a wav2vec-based ASR model. diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/docs/textless_s2st_real_data.md b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/docs/textless_s2st_real_data.md deleted file mode 100644 index ca6044be1ac9abf21cdcc91e1024ef64fb93cdcc..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/docs/textless_s2st_real_data.md +++ /dev/null @@ -1,89 +0,0 @@ -# Textless Speech-to-Speech Translation (S2ST) on Real Data - -We provide instructions and pre-trained models for the work "[Textless Speech-to-Speech Translation on Real Data (Lee et al. 2021)](https://arxiv.org/abs/2112.08352)". - -## Pre-trained Models - -### HuBERT -Model | Pretraining Data | Model | Quantizer -|---|---|---|--- -mHuBERT Base | [VoxPopuli](https://github.com/facebookresearch/voxpopuli) En, Es, Fr speech from the 100k subset | [download](https://dl.fbaipublicfiles.com/hubert/mhubert_base_vp_en_es_fr_it3.pt) | [L11 km1000](https://dl.fbaipublicfiles.com/hubert/mhubert_base_vp_en_es_fr_it3_L11_km1000.bin) - - -### Unit-based HiFi-GAN vocoder -Unit config | Unit size | Vocoder language | Dataset | Model -|---|---|---|---|--- -mHuBERT, layer 11 | 1000 | En | [LJSpeech](https://keithito.com/LJ-Speech-Dataset/) | [ckpt](https://dl.fbaipublicfiles.com/fairseq/speech_to_speech/vocoder/code_hifigan/mhubert_vp_en_es_fr_it3_400k_layer11_km1000_lj/g_00500000), [config](https://dl.fbaipublicfiles.com/fairseq/speech_to_speech/vocoder/code_hifigan/mhubert_vp_en_es_fr_it3_400k_layer11_km1000_lj/config.json) -mHuBERT, layer 11 | 1000 | Es | [CSS10](https://github.com/Kyubyong/css10) | [ckpt](https://dl.fbaipublicfiles.com/fairseq/speech_to_speech/vocoder/code_hifigan/mhubert_vp_en_es_fr_it3_400k_layer11_km1000_es_css10/g_00500000), [config](https://dl.fbaipublicfiles.com/fairseq/speech_to_speech/vocoder/code_hifigan/mhubert_vp_en_es_fr_it3_400k_layer11_km1000_es_css10/config.json) -mHuBERT, layer 11 | 1000 | Fr | [CSS10](https://github.com/Kyubyong/css10) | [ckpt](https://dl.fbaipublicfiles.com/fairseq/speech_to_speech/vocoder/code_hifigan/mhubert_vp_en_es_fr_it3_400k_layer11_km1000_fr_css10/g_00500000), [config](https://dl.fbaipublicfiles.com/fairseq/speech_to_speech/vocoder/code_hifigan/mhubert_vp_en_es_fr_it3_400k_layer11_km1000_fr_css10/config.json) - - -### Speech normalizer -Language | Training data | Target unit config | Model -|---|---|---|--- -En | 10 mins | mHuBERT, layer 11, km1000 | [download](https://dl.fbaipublicfiles.com/fairseq/speech_to_speech/speech_normalizer/en/en_10min.tar.gz) -En | 1 hr | mHuBERT, layer 11, km1000 | [download](https://dl.fbaipublicfiles.com/fairseq/speech_to_speech/speech_normalizer/en/en_1h.tar.gz) -En | 10 hrs | mHuBERT, layer 11, km1000 | [download](https://dl.fbaipublicfiles.com/fairseq/speech_to_speech/speech_normalizer/en/en_10h.tar.gz) -Es | 10 mins | mHuBERT, layer 11, km1000 | [download](https://dl.fbaipublicfiles.com/fairseq/speech_to_speech/speech_normalizer/es/es_10min.tar.gz) -Es | 1 hr | mHuBERT, layer 11, km1000 | [download](https://dl.fbaipublicfiles.com/fairseq/speech_to_speech/speech_normalizer/es/es_1h.tar.gz) -Es | 10 hrs | mHuBERT, layer 11, km1000 | [download](https://dl.fbaipublicfiles.com/fairseq/speech_to_speech/speech_normalizer/es/es_10h.tar.gz) -Fr | 10 mins | mHuBERT, layer 11, km1000 | [download](https://dl.fbaipublicfiles.com/fairseq/speech_to_speech/speech_normalizer/fr/fr_10min.tar.gz) -Fr | 1 hr | mHuBERT, layer 11, km1000 | [download](https://dl.fbaipublicfiles.com/fairseq/speech_to_speech/speech_normalizer/fr/fr_1h.tar.gz) -Fr | 10 hrs | mHuBERT, layer 11, km1000 | [download](https://dl.fbaipublicfiles.com/fairseq/speech_to_speech/speech_normalizer/fr/fr_10h.tar.gz) - -* Refer to the paper for the details of the training data. - -## Inference with Pre-trained Models - -### Speech normalizer -1. Download the pre-trained models, including the dictionary, to `DATA_DIR`. -2. Format the audio data. -```bash -# AUDIO_EXT: audio extension, e.g. wav, flac, etc. -# Assume all audio files are at ${AUDIO_DIR}/*.${AUDIO_EXT} - -python examples/speech_to_speech/preprocessing/prep_sn_data.py \ - --audio-dir ${AUDIO_DIR} --ext ${AUIDO_EXT} \ - --data-name ${GEN_SUBSET} --output-dir ${DATA_DIR} \ - --for-inference -``` - -3. Run the speech normalizer and post-process the output. -```bash -mkdir -p ${RESULTS_PATH} - -python examples/speech_recognition/new/infer.py \ - --config-dir examples/hubert/config/decode/ \ - --config-name infer_viterbi \ - task.data=${DATA_DIR} \ - task.normalize=false \ - common_eval.results_path=${RESULTS_PATH}/log \ - common_eval.path=${DATA_DIR}/checkpoint_best.pt \ - dataset.gen_subset=${GEN_SUBSET} \ - '+task.labels=["unit"]' \ - +decoding.results_path=${RESULTS_PATH} \ - common_eval.post_process=none \ - +dataset.batch_size=1 \ - common_eval.quiet=True - -# Post-process and generate output at ${RESULTS_PATH}/${GEN_SUBSET}.txt -python examples/speech_to_speech/preprocessing/prep_sn_output_data.py \ - --in-unit ${RESULTS_PATH}/hypo.units \ - --in-audio ${DATA_DIR}/${GEN_SUBSET}.tsv \ - --output-root ${RESULTS_PATH} -``` - - -### Unit-to-waveform conversion with unit vocoder -The pre-trained vocoders can support generating audio for both full unit sequences and reduced unit sequences (i.e. duplicating consecutive units removed). Set `--dur-prediction` for generating audio with reduced unit sequences. -```bash -# IN_CODE_FILE contains one unit sequence per line. Units are separated by space. - -python examples/speech_to_speech/generate_waveform_from_code.py \ - --in-code-file ${IN_CODE_FILE} \ - --vocoder ${VOCODER_CKPT} --vocoder-cfg ${VOCODER_CFG} \ - --results-path ${RESULTS_PATH} --dur-prediction -``` - -## Training new models -To be updated. diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/generate_waveform_from_code.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/generate_waveform_from_code.py deleted file mode 100644 index c57c07109418be830c6a47103397ab3bd08428da..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/generate_waveform_from_code.py +++ /dev/null @@ -1,114 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import json -import logging -import random -from pathlib import Path - -import soundfile as sf -import torch -from fairseq import utils -from fairseq.models.text_to_speech.vocoder import CodeHiFiGANVocoder -from tqdm import tqdm - -logging.basicConfig() -logging.root.setLevel(logging.INFO) -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - - -def dump_result(args, sample_id, pred_wav, suffix=""): - sf.write( - f"{args.results_path}/{sample_id}{suffix}_pred.wav", - pred_wav.detach().cpu().numpy(), - 16000, - ) - - -def load_code(in_file): - with open(in_file) as f: - out = [list(map(int, line.strip().split())) for line in f] - return out - - -def main(args): - logger.info(args) - - use_cuda = torch.cuda.is_available() and not args.cpu - - with open(args.vocoder_cfg) as f: - vocoder_cfg = json.load(f) - vocoder = CodeHiFiGANVocoder(args.vocoder, vocoder_cfg) - if use_cuda: - vocoder = vocoder.cuda() - - multispkr = vocoder.model.multispkr - if multispkr: - logger.info("multi-speaker vocoder") - num_speakers = vocoder_cfg.get( - "num_speakers", 200 - ) # following the default in codehifigan to set to 200 - assert ( - args.speaker_id < num_speakers - ), f"invalid --speaker-id ({args.speaker_id}) with total #speakers = {num_speakers}" - - data = load_code(args.in_code_file) - Path(args.results_path).mkdir(exist_ok=True, parents=True) - for i, d in tqdm(enumerate(data), total=len(data)): - x = { - "code": torch.LongTensor(d).view(1, -1), - } - suffix = "" - if multispkr: - spk = ( - random.randint(0, num_speakers - 1) - if args.speaker_id == -1 - else args.speaker_id - ) - suffix = f"_spk{spk}" - x["spkr"] = torch.LongTensor([spk]).view(1, 1) - - x = utils.move_to_cuda(x) if use_cuda else x - wav = vocoder(x, args.dur_prediction) - dump_result(args, i, wav, suffix=suffix) - - -def cli_main(): - parser = argparse.ArgumentParser() - parser.add_argument( - "--in-code-file", type=str, required=True, help="one unit sequence per line" - ) - parser.add_argument( - "--vocoder", type=str, required=True, help="path to the CodeHiFiGAN vocoder" - ) - parser.add_argument( - "--vocoder-cfg", - type=str, - required=True, - help="path to the CodeHiFiGAN vocoder config", - ) - parser.add_argument("--results-path", type=str, required=True) - parser.add_argument( - "--dur-prediction", - action="store_true", - help="enable duration prediction (for reduced/unique code sequences)", - ) - parser.add_argument( - "--speaker-id", - type=int, - default=-1, - help="Speaker id (for vocoder that supports multispeaker). Set to -1 to randomly sample speakers.", - ) - parser.add_argument("--cpu", action="store_true", help="run on CPU") - - args = parser.parse_args() - - main(args) - - -if __name__ == "__main__": - cli_main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/preprocessing/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/preprocessing/__init__.py deleted file mode 100644 index 6264236915a7269a4d920ee8213004374dd86a9a..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/preprocessing/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/preprocessing/data_utils.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/preprocessing/data_utils.py deleted file mode 100644 index a83a67f9549e79792e78d013a1dabccd78475185..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/preprocessing/data_utils.py +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from pathlib import Path -from typing import List, Optional - -from examples.speech_to_text.data_utils import S2TDataConfigWriter - - -def gen_config_yaml( - manifest_root: Path, - yaml_filename: str = "config.yaml", - specaugment_policy: Optional[str] = "lb", - feature_transform: Optional[List[str]] = None, - input_channels: Optional[int] = 1, - input_feat_per_channel: Optional[int] = 80, - audio_root: str = "", - vocoder_type: Optional[str] = None, - vocoder_checkpoint: Optional[str] = None, - vocoder_cfg: Optional[str] = None, - extra=None, -): - manifest_root = manifest_root.absolute() - writer = S2TDataConfigWriter(manifest_root / yaml_filename) - - if input_channels is not None: - writer.set_input_channels(input_channels) - if input_feat_per_channel is not None: - writer.set_input_feat_per_channel(input_feat_per_channel) - specaugment_setters = { - "lb": writer.set_specaugment_lb_policy, - "ld": writer.set_specaugment_ld_policy, - "sm": writer.set_specaugment_sm_policy, - "ss": writer.set_specaugment_ss_policy, - } - specaugment_setter = specaugment_setters.get(specaugment_policy, None) - if specaugment_setter is not None: - specaugment_setter() - - if feature_transform is None: - feature_transform = [] - else: - writer.set_feature_transforms("*", feature_transform) - - if specaugment_policy is not None: - writer.set_feature_transforms("_train", feature_transform + ["specaugment"]) - - if len(audio_root) > 0: - writer.set_audio_root(audio_root) - - if ( - vocoder_type is not None - and vocoder_checkpoint is not None - and vocoder_cfg is not None - ): - writer.set_extra( - { - "vocoder": { - "type": vocoder_type, - "config": vocoder_cfg, - "checkpoint": vocoder_checkpoint, - } - } - ) - - if extra is not None: - writer.set_extra(extra) - writer.flush() - - -def load_units(in_file): - out = {} - with open(in_file) as f: - for line in f: - sample_id, units = line.strip().split("|", 1) - out[sample_id] = units.split() - - return out - - -def process_units(units, reduce=False): - if not reduce: - return units - - out = [u for i, u in enumerate(units) if i == 0 or u != units[i - 1]] - return out diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/preprocessing/prep_s2spect_data.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/preprocessing/prep_s2spect_data.py deleted file mode 100644 index fc1a71ba2417c38f70f61da661bd62abdd17c451..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/preprocessing/prep_s2spect_data.py +++ /dev/null @@ -1,171 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import logging -import os -import shutil -from pathlib import Path - -import pandas as pd -import soundfile as sf -import torchaudio -from examples.speech_synthesis.data_utils import extract_logmel_spectrogram -from examples.speech_to_speech.preprocessing.data_utils import gen_config_yaml -from examples.speech_to_text.data_utils import ( - create_zip, - get_zip_manifest, - save_df_to_tsv, -) -from fairseq.data.audio.audio_utils import convert_waveform -from tqdm import tqdm - -logger = logging.getLogger(__name__) - -MANIFEST_COLUMNS = ["id", "src_audio", "src_n_frames", "tgt_audio", "tgt_n_frames"] - - -def prepare_target_data(args, tgt_audios): - feature_name = "logmelspec80" - zip_path = args.output_root / f"{feature_name}.zip" - if zip_path.exists(): - print(f"{zip_path} exists.") - return zip_path - - feature_root = args.output_root / feature_name - feature_root.mkdir(exist_ok=True) - - print("Extracting Mel spectrogram features...") - for tgt_audio in tqdm(tgt_audios): - sample_id = tgt_audio.stem - waveform, sample_rate = torchaudio.load(tgt_audio.as_posix()) - waveform, sample_rate = convert_waveform( - waveform, sample_rate, normalize_volume=args.normalize_volume, - to_sample_rate=args.sample_rate - ) - extract_logmel_spectrogram( - waveform, sample_rate, feature_root / f"{sample_id}.npy", - win_length=args.win_length, hop_length=args.hop_length, - n_fft=args.n_fft, n_mels=args.n_mels, f_min=args.f_min, - f_max=args.f_max - ) - print("ZIPing features...") - create_zip(feature_root, zip_path) - shutil.rmtree(feature_root) - - return zip_path - - -def process(args): - os.makedirs(args.output_root, exist_ok=True) - - manifest = {} - tgt_audios = [] - for split in args.data_split: - print(f"Processing {split}...") - - manifest[split] = {c: [] for c in MANIFEST_COLUMNS} - missing_tgt_audios = [] - src_audios = list(args.source_dir.glob(f"{split}/*.wav")) - for src_audio in tqdm(src_audios): - sample_id = src_audio.stem - - tgt_audio = args.target_dir / split / f"{sample_id}.wav" - if not tgt_audio.is_file(): - missing_tgt_audios.append(sample_id) - continue - - tgt_audios.append(tgt_audio) - - src_n_frames = sf.info(src_audio.as_posix()).frames - manifest[split]["id"].append(sample_id) - manifest[split]["src_audio"].append(src_audio.as_posix()) - manifest[split]["src_n_frames"].append( - src_n_frames // 160 - ) # estimation of 10-ms frame for 16kHz audio - - print(f"Processed {len(manifest[split]['id'])} samples") - if len(missing_tgt_audios) > 0: - print( - f"{len(missing_tgt_audios)} with missing target data (first 3 examples: {', '.join(missing_tgt_audios[:3])})" - ) - - # Extract features and pack features into ZIP - zip_path = prepare_target_data(args, tgt_audios) - - print("Fetching ZIP manifest...") - tgt_audio_paths, tgt_audio_lengths = get_zip_manifest(zip_path) - - print("Generating manifest...") - for split in args.data_split: - print(f"Processing {split}...") - - for sample_id in tqdm(manifest[split]["id"]): - manifest[split]["tgt_audio"].append(tgt_audio_paths[sample_id]) - manifest[split]["tgt_n_frames"].append(tgt_audio_lengths[sample_id]) - - out_manifest = args.output_root / f"{split}.tsv" - print(f"Writing manifest to {out_manifest}...") - save_df_to_tsv(pd.DataFrame.from_dict(manifest[split]), out_manifest) - - # Generate config YAML - win_len_t = args.win_length / args.sample_rate - hop_len_t = args.hop_length / args.sample_rate - extra = { - "features": { - "type": "spectrogram+melscale+log", - "sample_rate": args.sample_rate, - "eps": 1e-5, "n_mels": args.n_mels, "n_fft": args.n_fft, - "window_fn": "hann", "win_length": args.win_length, - "hop_length": args.hop_length, - "win_len_t": win_len_t, "hop_len_t": hop_len_t, - "f_min": args.f_min, "f_max": args.f_max, - "n_stft": args.n_fft // 2 + 1 - } - } - gen_config_yaml( - args.output_root, - audio_root=args.output_root.as_posix(), - specaugment_policy="lb", - feature_transform=["utterance_cmvn", "delta_deltas"], - extra=extra, - ) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument( - "--source-dir", required=True, type=Path, help="source audio directory" - ) - parser.add_argument( - "--target-dir", required=True, type=Path, help="target audio directory" - ) - parser.add_argument( - "--data-split", - default=["train", "valid", "test"], - nargs="+", - help="data split names", - ) - parser.add_argument( - "--output-root", required=True, type=Path, help="output directory" - ) - # target feature related - parser.add_argument("--win-length", type=int, default=1024) - parser.add_argument("--hop-length", type=int, default=256) - parser.add_argument("--n-fft", type=int, default=1024) - parser.add_argument("--n-mels", type=int, default=80) - parser.add_argument("--f-min", type=int, default=20) - parser.add_argument("--f-max", type=int, default=8000) - parser.add_argument("--sample-rate", type=int, default=22050) - parser.add_argument("--normalize-volume", "-n", action="store_true") - - args = parser.parse_args() - - process(args) - - -if __name__ == "__main__": - main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/preprocessing/prep_s2ut_data.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/preprocessing/prep_s2ut_data.py deleted file mode 100644 index 807b375f157472e0c69957da17c631c8043f22de..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/preprocessing/prep_s2ut_data.py +++ /dev/null @@ -1,113 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import logging -from pathlib import Path - -import pandas as pd -import soundfile as sf -from examples.speech_to_speech.preprocessing.data_utils import ( - gen_config_yaml, - load_units, - process_units, -) -from examples.speech_to_text.data_utils import save_df_to_tsv -from tqdm import tqdm - -logger = logging.getLogger(__name__) - -MANIFEST_COLUMNS = ["id", "src_audio", "src_n_frames", "tgt_audio", "tgt_n_frames"] - - -def process(args): - args.output_root.mkdir(exist_ok=True) - - print("Generating manifest...") - for split in args.data_split: - print(f"Processing {split}") - - # load target units - target_unit_data = load_units(args.target_dir / f"{split}.txt") - - manifest = {c: [] for c in MANIFEST_COLUMNS} - missing_tgt_audios = [] - src_audios = list(args.source_dir.glob(f"{split}/*.wav")) - for src_audio in tqdm(src_audios): - sample_id = src_audio.stem - - if sample_id not in target_unit_data: - missing_tgt_audios.append(sample_id) - continue - - src_n_frames = sf.info(src_audio.as_posix()).frames - manifest["id"].append(sample_id) - manifest["src_audio"].append(src_audio.as_posix()) - manifest["src_n_frames"].append( - src_n_frames // 160 - ) # estimation of 10-ms frame for 16kHz audio - - target_units = process_units(target_unit_data[sample_id], args.reduce_unit) - manifest["tgt_audio"].append(" ".join(target_units)) - manifest["tgt_n_frames"].append(len(target_units)) - - print(f"Processed {len(manifest['id'])} samples") - if len(missing_tgt_audios) > 0: - print( - f"{len(missing_tgt_audios)} with missing target data (first 3 examples: {', '.join(missing_tgt_audios[:3])})" - ) - - out_manifest = args.output_root / f"{split}.tsv" - print(f"Writing manifest to {out_manifest}...") - save_df_to_tsv(pd.DataFrame.from_dict(manifest), out_manifest) - - # Generate config YAML - gen_config_yaml( - args.output_root, - specaugment_policy="lb", - feature_transform=["utterance_cmvn"], - vocoder_type="code_hifigan", - vocoder_checkpoint=args.vocoder_checkpoint, - vocoder_cfg=args.vocoder_cfg, - ) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument( - "--source-dir", required=True, type=Path, help="source audio directory" - ) - parser.add_argument( - "--target-dir", required=True, type=Path, help="target audio directory" - ) - parser.add_argument( - "--data-split", - default=["train", "valid", "test"], - nargs="+", - help="data split names", - ) - parser.add_argument( - "--output-root", required=True, type=Path, help="output directory" - ) - parser.add_argument( - "--reduce-unit", - action="store_true", - help="reduce a target unit sequence to a unique unit sequence, i.e. '1 1 1 2 2' -> '1 2'", - ) - parser.add_argument( - "--vocoder-checkpoint", default=None, type=str, help="vocoder checkpoint" - ) - parser.add_argument( - "--vocoder-cfg", default=None, type=str, help="vocoder config file" - ) - - args = parser.parse_args() - - process(args) - - -if __name__ == "__main__": - main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/preprocessing/prep_sn_data.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/preprocessing/prep_sn_data.py deleted file mode 100644 index 4e9099e3438287a91d4c6d94b986399c2d903519..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/preprocessing/prep_sn_data.py +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. -# -# Adapted from examples/wav2vec/wav2vec_manifest.py -""" -Data preparation for the speech normalizer -""" - -import argparse -import glob -import os - -import soundfile -from examples.speech_to_speech.preprocessing.data_utils import load_units, process_units - - -def process(args): - assert ( - args.for_inference or args.target_unit is not None - ), "missing --target-unit or --for-inference" - - if not os.path.exists(args.output_dir): - os.makedirs(args.output_dir) - - dir_path = os.path.realpath(args.audio_dir) - search_path = os.path.join(dir_path, "**/*." + args.ext) - - if args.target_unit: - unit_data = load_units(args.target_unit) - - with open(os.path.join(args.output_dir, f"{args.data_name}.tsv"), "w") as o_t, open( - os.path.join(args.output_dir, f"{args.data_name}.unit"), "w" - ) as o_u: - print(dir_path, file=o_t) - for fname in glob.iglob(search_path, recursive=True): - file_path = os.path.realpath(fname) - frames = soundfile.info(fname).frames - print( - "{}\t{}".format(os.path.relpath(file_path, dir_path), frames), file=o_t - ) - - if args.for_inference: - print("0", file=o_u) - else: - sample_id = os.path.basename(file_path)[: -len(args.ext) - 1] - assert ( - sample_id in unit_data - ), f'{fname} does not have unit data in {args.target_unit}. Expecting sample_id "{sample_id}".' - target_units = process_units(unit_data[sample_id], reduce=True) - print(" ".join(target_units), file=o_u) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--audio-dir", required=True, type=str, help="audio directory") - parser.add_argument("--ext", default="flac", type=str, help="audio extension") - parser.add_argument( - "--data-name", - required=True, - type=str, - help="dataset name", - ) - parser.add_argument( - "--output-dir", required=True, type=str, help="output directory" - ) - parser.add_argument( - "--for-inference", - action="store_true", - help="set this if preparing data for running inference with a speech normalizer", - ) - parser.add_argument( - "--target-unit", - default=None, - type=str, - help="a file containing unit sequences in the format: sample_id|u1 u2 ...", - ) - - args = parser.parse_args() - - process(args) - - -if __name__ == "__main__": - main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/preprocessing/prep_sn_output_data.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/preprocessing/prep_sn_output_data.py deleted file mode 100644 index 06991343bd7ce9a3087a016d7e4bd04ca3a1fb97..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_speech/preprocessing/prep_sn_output_data.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -from pathlib import Path - -from tqdm import tqdm - - -def process(args): - args.output_root.mkdir(exist_ok=True) - - # load units - units = {} - with open(args.in_unit) as f: - for line in f: - unit_seq, utt_id = line.strip().rsplit(" ", 1) - utt_id = int(utt_id[6:-1]) # remove "(None-" - units[utt_id] = unit_seq - - with open(args.in_audio) as f, open( - args.output_root / f"{args.in_audio.stem}.txt", "w" - ) as o: - f.readline() - for i, line in enumerate(tqdm(f.readlines())): - audio, _ = line.strip().split("\t", 1) - sample_id = Path(audio).stem - o.write(f"{sample_id}|{units[i]}\n") - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument( - "--in-unit", - required=True, - type=Path, - help="unit file (output from the speech normalizer)", - ) - parser.add_argument( - "--in-audio", - required=True, - type=Path, - help="tsv file (input to the normalizer)", - ) - parser.add_argument( - "--output-root", required=True, type=Path, help="output directory" - ) - - args = parser.parse_args() - - process(args) - - -if __name__ == "__main__": - main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_text/README.md b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_text/README.md deleted file mode 100644 index f639d300d342f8de1392c98bfc44ec8690188539..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_text/README.md +++ /dev/null @@ -1,77 +0,0 @@ -# Speech-to-Text (S2T) Modeling - -[https://www.aclweb.org/anthology/2020.aacl-demo.6](https://www.aclweb.org/anthology/2020.aacl-demo.6.pdf) - -Speech recognition (ASR) and speech-to-text translation (ST) with fairseq. - -## Data Preparation -S2T modeling data consists of source speech features, target text and other optional information -(source text, speaker id, etc.). Fairseq S2T uses per-dataset-split TSV manifest files -to store these information. Each data field is represented by a column in the TSV file. - -Unlike text token embeddings, speech features (e.g. log mel-scale filter banks) are usually fixed -during model training and can be pre-computed. The manifest file contains the path to -either the feature file in NumPy format or the WAV/FLAC audio file. For the latter, -features will be extracted on-the-fly by fairseq S2T. Optionally, feature/audio files can be packed -into uncompressed ZIP files (then accessed via byte offset and length) to improve I/O performance. - -Fairseq S2T also employs a YAML file for data related configurations: tokenizer type and dictionary path -for the target text, feature transforms such as CMVN (cepstral mean and variance normalization) and SpecAugment, -temperature-based resampling, etc. - -## Model Training -Fairseq S2T uses the unified `fairseq-train` interface for model training. It requires arguments `--task speech_to_text`, - `--arch ` and `--config-yaml `. - -## Inference & Evaluation -Fairseq S2T uses the unified `fairseq-generate`/`fairseq-interactive` interface for inference and evaluation. It -requires arguments `--task speech_to_text` and `--config-yaml `. The interactive console takes -audio paths (one per line) as inputs. - - -## Examples -- [Speech Recognition (ASR) on LibriSpeech](docs/librispeech_example.md) - -- [Speech-to-Text Translation (ST) on MuST-C](docs/mustc_example.md) - -- [Speech-to-Text Translation (ST) on CoVoST 2](docs/covost_example.md) - -- [Speech-to-Text Translation (ST) on Multilingual TEDx](docs/mtedx_example.md) -- [Simultaneous Speech-to-Text Translation (SimulST) on MuST-C](docs/simulst_mustc_example.md) - -## Updates -- 02/04/2021: Added interactive decoding (`fairseq-interactive`) support. Examples: - [ASR (LibriSpeech)](docs/librispeech_example.md#interactive-decoding) - and [ST (CoVoST 2)](docs/covost_example.md#interactive-decoding). -- 01/08/2021: Several fixes for S2T Transformer model, inference-time de-tokenization, scorer configuration and data - preparation scripts. We also add pre-trained models to the examples and revise the instructions. - Breaking changes: the data preparation scripts now extract filterbank features without CMVN. CMVN is instead applied - on-the-fly (defined in the config YAML). - -## What's Next -- We are migrating the old fairseq [ASR example](../speech_recognition) into this S2T framework and - merging the features from both sides. -- The following papers also base their experiments on fairseq S2T. We are adding more examples for replication. - - [Improving Cross-Lingual Transfer Learning for End-to-End Speech Recognition with Speech Translation (Wang et al., 2020)](https://arxiv.org/abs/2006.05474) - - [Self-Supervised Representations Improve End-to-End Speech Translation (Wu et al., 2020)](https://arxiv.org/abs/2006.12124) - - [Self-Training for End-to-End Speech Translation (Pino et al., 2020)](https://arxiv.org/abs/2006.02490) - - [CoVoST: A Diverse Multilingual Speech-To-Text Translation Corpus (Wang et al., 2020)](https://arxiv.org/abs/2002.01320) - - [Harnessing Indirect Training Data for End-to-End Automatic Speech Translation: Tricks of the Trade (Pino et al., 2019)](https://arxiv.org/abs/1909.06515) - -## Citation -Please cite as: -``` -@inproceedings{wang2020fairseqs2t, - title = {fairseq S2T: Fast Speech-to-Text Modeling with fairseq}, - author = {Changhan Wang and Yun Tang and Xutai Ma and Anne Wu and Dmytro Okhonko and Juan Pino}, - booktitle = {Proceedings of the 2020 Conference of the Asian Chapter of the Association for Computational Linguistics (AACL): System Demonstrations}, - year = {2020}, -} - -@inproceedings{ott2019fairseq, - title = {fairseq: A Fast, Extensible Toolkit for Sequence Modeling}, - author = {Myle Ott and Sergey Edunov and Alexei Baevski and Angela Fan and Sam Gross and Nathan Ng and David Grangier and Michael Auli}, - booktitle = {Proceedings of NAACL-HLT 2019: Demonstrations}, - year = {2019}, -} -``` diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_text/data_utils.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_text/data_utils.py deleted file mode 100644 index 69b0552a479b1f67f85fe6c4aea9ba99a6e17b0c..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_text/data_utils.py +++ /dev/null @@ -1,385 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import csv -import io -import zipfile -from functools import reduce -from multiprocessing import cpu_count -from pathlib import Path -from typing import Any, Dict, List, Optional, Union - -import numpy as np -import pandas as pd -import sentencepiece as sp -import soundfile as sf -import torch -from fairseq.data.audio.audio_utils import ( - _get_kaldi_fbank, - _get_torchaudio_fbank, - convert_waveform, - is_npy_data, - is_sf_audio_data, -) -from tqdm import tqdm - -UNK_TOKEN, UNK_TOKEN_ID = "", 3 -BOS_TOKEN, BOS_TOKEN_ID = "", 0 -EOS_TOKEN, EOS_TOKEN_ID = "", 2 -PAD_TOKEN, PAD_TOKEN_ID = "", 1 - - -def gen_vocab( - input_path: Path, output_path_prefix: Path, model_type="bpe", - vocab_size=1000, special_symbols: Optional[List[str]] = None -): - # Train SentencePiece Model - arguments = [ - f"--input={input_path.as_posix()}", - f"--model_prefix={output_path_prefix.as_posix()}", - f"--model_type={model_type}", - f"--vocab_size={vocab_size}", - "--character_coverage=1.0", - f"--num_threads={cpu_count()}", - f"--unk_id={UNK_TOKEN_ID}", - f"--bos_id={BOS_TOKEN_ID}", - f"--eos_id={EOS_TOKEN_ID}", - f"--pad_id={PAD_TOKEN_ID}", - ] - if special_symbols is not None: - _special_symbols = ",".join(special_symbols) - arguments.append(f"--user_defined_symbols={_special_symbols}") - sp.SentencePieceTrainer.Train(" ".join(arguments)) - # Export fairseq dictionary - spm = sp.SentencePieceProcessor() - spm.Load(output_path_prefix.as_posix() + ".model") - vocab = {i: spm.IdToPiece(i) for i in range(spm.GetPieceSize())} - assert ( - vocab.get(UNK_TOKEN_ID) == UNK_TOKEN - and vocab.get(PAD_TOKEN_ID) == PAD_TOKEN - and vocab.get(BOS_TOKEN_ID) == BOS_TOKEN - and vocab.get(EOS_TOKEN_ID) == EOS_TOKEN - ) - vocab = { - i: s - for i, s in vocab.items() - if s not in {UNK_TOKEN, BOS_TOKEN, EOS_TOKEN, PAD_TOKEN} - } - with open(output_path_prefix.as_posix() + ".txt", "w") as f_out: - for _, s in sorted(vocab.items(), key=lambda x: x[0]): - f_out.write(f"{s} 1\n") - - -def extract_fbank_features( - waveform: torch.FloatTensor, - sample_rate: int, - output_path: Optional[Path] = None, - n_mel_bins: int = 80, - overwrite: bool = False, -): - if output_path is not None and output_path.is_file() and not overwrite: - return - - _waveform, _ = convert_waveform(waveform, sample_rate, to_mono=True) - # Kaldi compliance: 16-bit signed integers - _waveform = _waveform * (2 ** 15) - _waveform = _waveform.numpy() - - features = _get_kaldi_fbank(_waveform, sample_rate, n_mel_bins) - if features is None: - features = _get_torchaudio_fbank(_waveform, sample_rate, n_mel_bins) - if features is None: - raise ImportError( - "Please install pyKaldi or torchaudio to enable fbank feature extraction" - ) - - if output_path is not None: - np.save(output_path.as_posix(), features) - return features - - -def create_zip(data_root: Path, zip_path: Path): - paths = list(data_root.glob("*.npy")) - paths.extend(data_root.glob("*.flac")) - with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_STORED) as f: - for path in tqdm(paths): - f.write(path, arcname=path.name) - - -def get_zip_manifest( - zip_path: Path, zip_root: Optional[Path] = None, is_audio=False -): - _zip_path = Path.joinpath(zip_root or Path(""), zip_path) - with zipfile.ZipFile(_zip_path, mode="r") as f: - info = f.infolist() - paths, lengths = {}, {} - for i in tqdm(info): - utt_id = Path(i.filename).stem - offset, file_size = i.header_offset + 30 + len(i.filename), i.file_size - paths[utt_id] = f"{zip_path.as_posix()}:{offset}:{file_size}" - with open(_zip_path, "rb") as f: - f.seek(offset) - byte_data = f.read(file_size) - assert len(byte_data) > 1 - if is_audio: - assert is_sf_audio_data(byte_data), i - else: - assert is_npy_data(byte_data), i - byte_data_fp = io.BytesIO(byte_data) - if is_audio: - lengths[utt_id] = sf.info(byte_data_fp).frames - else: - lengths[utt_id] = np.load(byte_data_fp).shape[0] - return paths, lengths - - -def gen_config_yaml( - manifest_root: Path, - spm_filename: Optional[str] = None, - vocab_name: Optional[str] = None, - yaml_filename: str = "config.yaml", - specaugment_policy: Optional[str] = "lb", - prepend_tgt_lang_tag: bool = False, - sampling_alpha: Optional[float] = None, - input_channels: Optional[int] = 1, - input_feat_per_channel: Optional[int] = 80, - audio_root: str = "", - cmvn_type: str = "utterance", - gcmvn_path: Optional[Path] = None, - extra=None -): - manifest_root = manifest_root.absolute() - writer = S2TDataConfigWriter(manifest_root / yaml_filename) - assert spm_filename is not None or vocab_name is not None - vocab_name = spm_filename.replace(".model", ".txt") if vocab_name is None \ - else vocab_name - writer.set_vocab_filename(vocab_name) - if input_channels is not None: - writer.set_input_channels(input_channels) - if input_feat_per_channel is not None: - writer.set_input_feat_per_channel(input_feat_per_channel) - specaugment_setters = { - "lb": writer.set_specaugment_lb_policy, - "ld": writer.set_specaugment_ld_policy, - "sm": writer.set_specaugment_sm_policy, - "ss": writer.set_specaugment_ss_policy, - } - specaugment_setter = specaugment_setters.get(specaugment_policy, None) - if specaugment_setter is not None: - specaugment_setter() - if spm_filename is not None: - writer.set_bpe_tokenizer( - { - "bpe": "sentencepiece", - "sentencepiece_model": (manifest_root / spm_filename).as_posix(), - } - ) - if prepend_tgt_lang_tag: - writer.set_prepend_tgt_lang_tag(True) - if sampling_alpha is not None: - writer.set_sampling_alpha(sampling_alpha) - - if cmvn_type not in ["global", "utterance"]: - raise NotImplementedError - - if specaugment_policy is not None: - writer.set_feature_transforms( - "_train", [f"{cmvn_type}_cmvn", "specaugment"] - ) - writer.set_feature_transforms("*", [f"{cmvn_type}_cmvn"]) - - if cmvn_type == "global": - if gcmvn_path is None: - raise ValueError("Please provide path of global cmvn file.") - else: - writer.set_global_cmvn(gcmvn_path.as_posix()) - - if len(audio_root) > 0: - writer.set_audio_root(audio_root) - - if extra is not None: - writer.set_extra(extra) - writer.flush() - - -def load_df_from_tsv(path: Union[str, Path]) -> pd.DataFrame: - _path = path if isinstance(path, str) else path.as_posix() - return pd.read_csv( - _path, - sep="\t", - header=0, - encoding="utf-8", - escapechar="\\", - quoting=csv.QUOTE_NONE, - na_filter=False, - ) - - -def save_df_to_tsv(dataframe, path: Union[str, Path]): - _path = path if isinstance(path, str) else path.as_posix() - dataframe.to_csv( - _path, - sep="\t", - header=True, - index=False, - encoding="utf-8", - escapechar="\\", - quoting=csv.QUOTE_NONE, - ) - - -def load_tsv_to_dicts(path: Union[str, Path]) -> List[dict]: - with open(path, "r") as f: - reader = csv.DictReader( - f, - delimiter="\t", - quotechar=None, - doublequote=False, - lineterminator="\n", - quoting=csv.QUOTE_NONE, - ) - rows = [dict(e) for e in reader] - return rows - - -def filter_manifest_df( - df, is_train_split=False, extra_filters=None, min_n_frames=5, max_n_frames=3000 -): - filters = { - "no speech": df["audio"] == "", - f"short speech (<{min_n_frames} frames)": df["n_frames"] < min_n_frames, - "empty sentence": df["tgt_text"] == "", - } - if is_train_split: - filters[f"long speech (>{max_n_frames} frames)"] = df["n_frames"] > max_n_frames - if extra_filters is not None: - filters.update(extra_filters) - invalid = reduce(lambda x, y: x | y, filters.values()) - valid = ~invalid - print( - "| " - + ", ".join(f"{n}: {f.sum()}" for n, f in filters.items()) - + f", total {invalid.sum()} filtered, {valid.sum()} remained." - ) - return df[valid] - - -def cal_gcmvn_stats(features_list): - features = np.concatenate(features_list) - square_sums = (features ** 2).sum(axis=0) - mean = features.mean(axis=0) - features = np.subtract(features, mean) - var = square_sums / features.shape[0] - mean ** 2 - std = np.sqrt(np.maximum(var, 1e-8)) - return {"mean": mean.astype("float32"), "std": std.astype("float32")} - - -class S2TDataConfigWriter(object): - DEFAULT_VOCAB_FILENAME = "dict.txt" - DEFAULT_INPUT_FEAT_PER_CHANNEL = 80 - DEFAULT_INPUT_CHANNELS = 1 - - def __init__(self, yaml_path: Path): - try: - import yaml - except ImportError: - print("Please install PyYAML for S2T data config YAML files") - self.yaml = yaml - self.yaml_path = yaml_path - self.config = {} - - def flush(self): - with open(self.yaml_path, "w") as f: - self.yaml.dump(self.config, f) - - def set_audio_root(self, audio_root=""): - self.config["audio_root"] = audio_root - - def set_vocab_filename(self, vocab_filename: str = "dict.txt"): - self.config["vocab_filename"] = vocab_filename - - def set_specaugment( - self, - time_wrap_w: int, - freq_mask_n: int, - freq_mask_f: int, - time_mask_n: int, - time_mask_t: int, - time_mask_p: float, - ): - self.config["specaugment"] = { - "time_wrap_W": time_wrap_w, - "freq_mask_N": freq_mask_n, - "freq_mask_F": freq_mask_f, - "time_mask_N": time_mask_n, - "time_mask_T": time_mask_t, - "time_mask_p": time_mask_p, - } - - def set_specaugment_lb_policy(self): - self.set_specaugment( - time_wrap_w=0, - freq_mask_n=1, - freq_mask_f=27, - time_mask_n=1, - time_mask_t=100, - time_mask_p=1.0, - ) - - def set_specaugment_ld_policy(self): - self.set_specaugment( - time_wrap_w=0, - freq_mask_n=2, - freq_mask_f=27, - time_mask_n=2, - time_mask_t=100, - time_mask_p=1.0, - ) - - def set_specaugment_sm_policy(self): - self.set_specaugment( - time_wrap_w=0, - freq_mask_n=2, - freq_mask_f=15, - time_mask_n=2, - time_mask_t=70, - time_mask_p=0.2, - ) - - def set_specaugment_ss_policy(self): - self.set_specaugment( - time_wrap_w=0, - freq_mask_n=2, - freq_mask_f=27, - time_mask_n=2, - time_mask_t=70, - time_mask_p=0.2, - ) - - def set_input_channels(self, input_channels: int = 1): - self.config["input_channels"] = input_channels - - def set_input_feat_per_channel(self, input_feat_per_channel: int = 80): - self.config["input_feat_per_channel"] = input_feat_per_channel - - def set_bpe_tokenizer(self, bpe_tokenizer: Dict[str, Any]): - self.config["bpe_tokenizer"] = bpe_tokenizer - - def set_global_cmvn(self, stats_npz_path: str): - self.config["global_cmvn"] = {"stats_npz_path": stats_npz_path} - - def set_feature_transforms(self, split: str, transforms: List[str]): - if "transforms" not in self.config: - self.config["transforms"] = {} - self.config["transforms"][split] = transforms - - def set_prepend_tgt_lang_tag(self, flag: bool = True): - self.config["prepend_tgt_lang_tag"] = flag - - def set_sampling_alpha(self, sampling_alpha: float = 1.0): - self.config["sampling_alpha"] = sampling_alpha - - def set_extra(self, data): - self.config.update(data) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_text/docs/covost_example.md b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_text/docs/covost_example.md deleted file mode 100644 index 6282428b16496dbc8512da2e9cd3204a065c73a5..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_text/docs/covost_example.md +++ /dev/null @@ -1,140 +0,0 @@ -[[Back]](..) - -# S2T Example: ST on CoVoST - -We replicate the experiments in -[CoVoST 2 and Massively Multilingual Speech-to-Text Translation (Wang et al., 2020)](https://arxiv.org/abs/2007.10310). - -## Data Preparation - -[Download](https://commonvoice.mozilla.org/en/datasets) and unpack Common Voice v4 to a path -`${COVOST_ROOT}/${SOURCE_LANG_ID}`, then preprocess it with - -```bash -# additional Python packages for S2T data processing/model training -pip install pandas torchaudio sentencepiece - -# En ASR -python examples/speech_to_text/prep_covost_data.py \ - --data-root ${COVOST_ROOT} --vocab-type char --src-lang en -# ST -python examples/speech_to_text/prep_covost_data.py \ - --data-root ${COVOST_ROOT} --vocab-type char \ - --src-lang fr --tgt-lang en -``` - -The generated files (manifest, features, vocabulary and data configuration) will be added to -`${COVOST_ROOT}/${SOURCE_LANG_ID}`. - -Download our vocabulary files if you want to use our pre-trained models: - -- ASR: [En](https://dl.fbaipublicfiles.com/fairseq/s2t/covost2_en_asr_vocab_char.zip) -- ST: [Fr-En](https://dl.fbaipublicfiles.com/fairseq/s2t/covost2_fr_en_st_vocab_char.zip), [De-En](https://dl.fbaipublicfiles.com/fairseq/s2t/covost2_de_en_st_vocab_char.zip), [Es-En](https://dl.fbaipublicfiles.com/fairseq/s2t/covost2_es_en_st_vocab_char.zip), [Ca-En](https://dl.fbaipublicfiles.com/fairseq/s2t/covost2_ca_en_st_vocab_char.zip), [En-De](https://dl.fbaipublicfiles.com/fairseq/s2t/covost2_en_de_st_vocab_char.zip), [En-Ca](https://dl.fbaipublicfiles.com/fairseq/s2t/covost2_en_ca_st_vocab_char.zip), [En-Fa](https://dl.fbaipublicfiles.com/fairseq/s2t/covost2_en_fa_st_vocab_char.zip), [En-Et](https://dl.fbaipublicfiles.com/fairseq/s2t/covost2_en_et_st_vocab_char.zip) - -## ASR - -#### Training - -We train an En ASR model for encoder pre-training some of the ST models. - -```bash -fairseq-train ${COVOST_ROOT}/en \ - --config-yaml config_asr_en.yaml --train-subset train_asr_en --valid-subset dev_asr_en \ - --save-dir ${ASR_SAVE_DIR} --num-workers 4 --max-tokens 50000 --max-update 60000 \ - --task speech_to_text --criterion label_smoothed_cross_entropy --label-smoothing 0.1 \ - --report-accuracy --arch s2t_transformer_s --dropout 0.15 --optimizer adam --lr 2e-3 \ - --lr-scheduler inverse_sqrt --warmup-updates 10000 --clip-norm 10.0 --seed 1 --update-freq 8 \ - --attn-type None --pos-enc-type ${POS_ENC_TYPE} -``` - -where `ASR_SAVE_DIR` is the checkpoint root path and `POS_ENC_TYPE` refers to positional encoding to be used in the conformer encoder. -Set it to `abs`, `rope` or `rel_pos` to use the absolute positional encoding, rotary positional encoding or relative positional encoding in the conformer layer respectively. -Transformer encoder only supports absolute positional encoding and by default, the transformer encoder will be used. -To switch to conformer, set `--attn-type espnet` and `--POS_ENC_TYPE`. We set `--update-freq 8` to simulate 8 GPUs with 1 GPU. You may want to update it accordingly when using more than 1 GPU. - -#### Inference & Evaluation - -```bash -CHECKPOINT_FILENAME=avg_last_10_checkpoint.pt -python scripts/average_checkpoints.py \ - --inputs ${ASR_SAVE_DIR} --num-epoch-checkpoints 10 \ - --output "${ASR_SAVE_DIR}/${CHECKPOINT_FILENAME}" -fairseq-generate ${COVOST_ROOT}/en \ - --config-yaml config_asr_en.yaml --gen-subset test_asr_en --task speech_to_text \ - --path ${ASR_SAVE_DIR}/${CHECKPOINT_FILENAME} --max-tokens 50000 --beam 5 \ - --scoring wer --wer-tokenizer 13a --wer-lowercase --wer-remove-punct -``` - -#### Results - -| --arch | --pos-enc-type | Params | En | Model | -|---|---|---|---|---| -| s2t_transformer_s | - | 31M | 25.6 | [Download](https://dl.fbaipublicfiles.com/fairseq/s2t/covost2_en_asr_transformer_s.pt) | -| s2t_conformer | rel_pos | 42.9M | 23.18| [Download](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/en_asr/rel_pos_asr_checkpoint_best.pt) | -| s2t_conformer | rope | 42.1M | 23.8| [Download](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/en_asr/rope_pos_asr_checkpoint_best.pt) | -| s2t_conformer | abs | 42.1M | 23.8| [Download](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/en_asr/abs_asr_checkpoint_best.pt) | - -## ST - -#### Training - -Fr-En as example: - -```bash -fairseq-train ${COVOST_ROOT}/fr \ - --config-yaml config_st_fr_en.yaml --train-subset train_st_fr_en --valid-subset dev_st_fr_en \ - --save-dir ${ST_SAVE_DIR} --num-workers 4 --max-update 30000 --max-tokens 40000 \ # --max-tokens 50000 for en-* - --task speech_to_text --criterion label_smoothed_cross_entropy --label-smoothing 0.1 --report-accuracy \ - --arch s2t_transformer_s --encoder-freezing-updates 1000 --optimizer adam --lr 2e-3 \ - --lr-scheduler inverse_sqrt --warmup-updates 10000 --clip-norm 10.0 --seed 1 --update-freq 8 \ - --attn-type None --pos-enc-type ${POS_ENC_TYPE} \ - --load-pretrained-encoder-from ${ASR_SAVE_DIR}/${CHECKPOINT_FILENAME} -``` - -where `ST_SAVE_DIR` is the checkpoint root path and `POS_ENC_TYPE` refers to positional encoding to be used in the conformer encoder. -Set it to `abs`, `rope` or `rel_pos` to use the absolute positional encoding, rotary positional encoding or relative positional encoding in the conformer layer respectively. -Transformer encoder only supports absolute positional encoding and by default, the transformer encoder will be used. -To switch to conformer, set `--attn-type espnet` and `--POS_ENC_TYPE`. Optionally load the pre-trained En ASR encoder for faster training and better -performance: `--load-pretrained-encoder-from `. We set `--update-freq 8` to simulate 8 GPUs with 1 GPU. -You may want to update it accordingly when using more than 1 GPU. - -#### Inference & Evaluation - -Average the last 10 checkpoints and evaluate on test split: - -```bash -CHECKPOINT_FILENAME=avg_last_10_checkpoint.pt -python scripts/average_checkpoints.py \ - --inputs ${ST_SAVE_DIR} --num-epoch-checkpoints 10 \ - --output "${ST_SAVE_DIR}/${CHECKPOINT_FILENAME}" -fairseq-generate ${COVOST_ROOT}/fr \ - --config-yaml config_st_fr_en.yaml --gen-subset test_st_fr_en --task speech_to_text \ - --path ${ST_SAVE_DIR}/${CHECKPOINT_FILENAME} \ - --max-tokens 50000 --beam 5 --scoring sacrebleu -``` - -## Interactive Decoding - -Launch the interactive console via - -```bash -fairseq-interactive ${COVOST_ROOT}/fr --config-yaml config_st_fr_en.yaml \ - --task speech_to_text --path ${SAVE_DIR}/${CHECKPOINT_FILENAME} \ - --max-tokens 50000 --beam 5 -``` - -Type in WAV/FLAC/OGG audio paths (one per line) after the prompt. - -#### Results - -| --arch | --pos-enc-type | Params | ASR PT | Fr-En | De-En | Es-En | Ca-En | En-De | En-Ca | En-Fa | En-Et | Model | -|---|---|---|---|---|---|---|---|---|---|---|---|---| -| s2t_transformer | - | 31M | Yes | [27.2](https://dl.fbaipublicfiles.com/fairseq/s2t/covost2_fr_en_st_transformer_s.pt) | [17.7](https://dl.fbaipublicfiles.com/fairseq/s2t/covost2_de_en_st_transformer_s.pt) | [23.1](https://dl.fbaipublicfiles.com/fairseq/s2t/covost2_es_en_st_transformer_s.pt) | [19.3](https://dl.fbaipublicfiles.com/fairseq/s2t/covost2_ca_en_st_transformer_s.pt) | [16.1](https://dl.fbaipublicfiles.com/fairseq/s2t/covost2_en_de_st_transformer_s.pt) | [21.6](https://dl.fbaipublicfiles.com/fairseq/s2t/covost2_en_ca_st_transformer_s.pt) | [12.9](https://dl.fbaipublicfiles.com/fairseq/s2t/covost2_en_fa_st_transformer_s.pt) | [12.8](https://dl.fbaipublicfiles.com/fairseq/s2t/covost2_en_et_st_transformer_s.pt) | (<-Download) | -| s2t_conformer | rel_pos | 42.9M | No | [28.32](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/fr_en/rel_pos_from_scratch_avg_last_10_checkpoint.pt) | [18.21](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/de_en/rel_pos_from_scratch_avg_last_10_checkpoint.pt) | [25.98](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/es_en/rel_pos_from_scratch_avg_last_10_checkpoint.pt) | [21.13](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/ca_en/rel_pos_from_scratch_avg_last_10_checkpoint.pt) | [20.37](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/en_de/rel_pos_from_scratch_avg_last_10_checkpoint.pt) | [25.89](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/en_ca/rel_pos_from_scratch_avg_last_10_checkpoint.pt) | [15.59](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/en_fa/rel_pos_from_scratch_avg_last_10_checkpoint.pt) | [14.49](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/en_et/rel_pos_from_scratch_avg_last_10_checkpoint.pt) | (<-Download) | -| s2t_conformer | rel_pos | 42.9M | Yes| [27.15](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/fr_en/rel_pos_asr_pt_avg_last_10_checkpoint.pt) | [18.22](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/de_en/rel_pos_asr_pt_avg_last_10_checkpoint.pt) | [25.14](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/es_en/rel_pos_asr_pt_avg_last_10_checkpoint.pt) | [21.68](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/ca_en/rel_pos_asr_pt_avg_last_10_checkpoint.pt) | [20.35](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/en_de/rel_pos_asr_pt_avg_last_10_checkpoint.pt) | [25.92](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/en_ca/rel_pos_asr_pt_avg_last_10_checkpoint.pt) | [15.76](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/en_fa/rel_pos_asr_pt_avg_last_10_checkpoint.pt) | [16.52](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/en_et/rel_pos_asr_pt_avg_last_10_checkpoint.pt) | (<-Download) | -| s2t_conformer | rope | 42.1M | No | [27.61](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/fr_en/rope_from_scratch_avg_last_10_checkpoint.pt) | [17.6](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/de_en/rope_from_scratch_avg_last_10_checkpoint.pt) | [24.91](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/es_en/rope_from_scratch_avg_last_10_checkpoint.pt) | [20.78](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/ca_en/rope_from_scratch_avg_last_10_checkpoint.pt) | [19.7](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/en_de/rope_from_scratch_avg_last_10_checkpoint.pt) | [25.13](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/en_ca/rope_from_scratch_avg_last_10_checkpoint.pt) | [15.22](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/en_fa/rope_from_scratch_avg_last_10_checkpoint.pt) | [15.87](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/en_et/rope_from_scratch_avg_last_10_checkpoint.pt) | (<-Download) | -| s2t_conformer | rope | 42.1M | Yes | [26.99](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/fr_en/rope_asr_pt_avg_last_10_checkpoint.pt) | [17.71](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/de_en/rope_asr_pt_avg_last_10_checkpoint.pt) | [24.24](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/es_en/rope_asr_pt_avg_last_10_checkpoint.pt) | [21.24](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/ca_en/rope_asr_pt_avg_last_10_checkpoint.pt) | [19.9](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/en_de/rope_asr_pt_avg_last_10_checkpoint.pt) | [25.25](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/en_ca/rope_asr_pt_avg_last_10_checkpoint.pt) | [15.58](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/en_fa/rope_asr_pt_avg_last_10_checkpoint.pt) | [15.97](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/en_et/rope_asr_pt_avg_last_10_checkpoint.pt) | (<-Download) | -| s2t_conformer | abs | 42.1M | No | [27.45](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/fr_en/abs_from_scratch_avg_last_10_checkpoint.pt) | [17.25](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/de_en/abs_from_scratch_avg_last_10_checkpoint.pt) | [25.01](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/es_en/abs_from_scratch_avg_last_10_checkpoint.pt) | [20.26](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/ca_en/abs_from_scratch_avg_last_10_checkpoint.pt) | [19.86](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/en_de/abs_from_scratch_avg_last_10_checkpoint.pt) | [25.25](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/en_ca/abs_from_scratch_avg_last_10_checkpoint.pt) | [15.46](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/en_fa/abs_from_scratch_avg_last_10_checkpoint.pt) | [15.81](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/en_et/abs_from_scratch_avg_last_10_checkpoint.pt) | (<-Download) | -| s2t_conforme | abs | 42.1M | Yes| [26.52](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/fr_en/abs_asr_pt_avg_last_10_checkpoint.pt) | [17.37](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/de_en/abs_asr_pt_avg_last_10_checkpoint.pt) | [25.40](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/es_en/abs_asr_pt_avg_last_10_checkpoint.pt) | [20.45](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/ca_en/abs_asr_pt_avg_last_10_checkpoint.pt) | [19.57](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/en_de/abs_asr_pt_avg_last_10_checkpoint.pt) | [25.40](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/en_ca/abs_asr_pt_avg_last_10_checkpoint.pt) | [15.17](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/en_fa/abs_asr_pt_avg_last_10_checkpoint.pt) | [15.83](https://dl.fbaipublicfiles.com/fairseq/conformer/covost2/en_et/abs_asr_pt_avg_last_10_checkpoint.pt) | (<-Download) | - -[[Back]](..) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_text/docs/librispeech_example.md b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_text/docs/librispeech_example.md deleted file mode 100644 index 4040fda9426027537036ba987d087a43e734bfd9..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_text/docs/librispeech_example.md +++ /dev/null @@ -1,69 +0,0 @@ -[[Back]](..) - -# S2T Example: Speech Recognition (ASR) on LibriSpeech -[LibriSpeech](https://www.danielpovey.com/files/2015_icassp_librispeech.pdf) is a de-facto standard English ASR -benchmark. We provide competitive -vanilla [Transformer](https://papers.nips.cc/paper/2017/file/3f5ee243547dee91fbd053c1c4a845aa-Paper.pdf) baselines. - -## Data preparation -Download and preprocess LibriSpeech data with -```bash -# additional Python packages for S2T data processing/model training -pip install pandas torchaudio sentencepiece - -python examples/speech_to_text/prep_librispeech_data.py \ - --output-root ${LS_ROOT} --vocab-type unigram --vocab-size 10000 -``` -where `LS_ROOT` is the root path for downloaded data as well as generated files (manifest, features, vocabulary and -data configuration). - -[Download](https://dl.fbaipublicfiles.com/fairseq/s2t/librispeech_vocab_unigram10000.zip) our vocabulary files -if you want to use our pre-trained models. - -## Training -```bash -fairseq-train ${LS_ROOT} --save-dir ${SAVE_DIR} \ - --config-yaml config.yaml --train-subset train-clean-100,train-clean-360,train-other-500 --valid-subset dev-clean,dev-other \ - --num-workers 4 --max-tokens 40000 --max-update 300000 \ - --task speech_to_text --criterion label_smoothed_cross_entropy --label-smoothing 0.1 --report-accuracy \ - --arch s2t_transformer_s --share-decoder-input-output-embed \ - --optimizer adam --lr 2e-3 --lr-scheduler inverse_sqrt --warmup-updates 10000 \ - --clip-norm 10.0 --seed 1 --update-freq 8 -``` -where `SAVE_DIR` is the checkpoint root path. Here we use `--arch s2t_transformer_s` (31M parameters) as example. -For better performance, you may switch to `s2t_transformer_m` (71M, with `--lr 1e-3`) or `s2t_transformer_l` -(268M, with `--lr 5e-4`). We set `--update-freq 8` to simulate 8 GPUs with 1 GPU. You may want to update it accordingly -when using more than 1 GPU. - -## Inference & Evaluation -Average the last 10 checkpoints and evaluate on the 4 splits -(`dev-clean`, `dev-other`, `test-clean` and `test-other`): -```bash -CHECKPOINT_FILENAME=avg_last_10_checkpoint.pt -python scripts/average_checkpoints.py --inputs ${SAVE_DIR} \ - --num-epoch-checkpoints 10 \ - --output "${SAVE_DIR}/${CHECKPOINT_FILENAME}" -for SUBSET in dev-clean dev-other test-clean test-other; do - fairseq-generate ${LS_ROOT} --config-yaml config.yaml --gen-subset ${SUBSET} \ - --task speech_to_text --path ${SAVE_DIR}/${CHECKPOINT_FILENAME} \ - --max-tokens 50000 --beam 5 --scoring wer -done -``` - -## Interactive Decoding -Launch the interactive console via -```bash -fairseq-interactive ${LS_ROOT} --config-yaml config.yaml --task speech_to_text \ - --path ${SAVE_DIR}/${CHECKPOINT_FILENAME} --max-tokens 50000 --beam 5 -``` -Type in WAV/FLAC/OGG audio paths (one per line) after the prompt. - -## Results - -| --arch | Params | dev-clean | dev-other | test-clean | test-other | Model | -|---|---|---|---|---|---|---| -| s2t_transformer_s | 30M | 3.8 | 8.9 | 4.4 | 9.0 | [Download](https://dl.fbaipublicfiles.com/fairseq/s2t/librispeech_transformer_s.pt) | -| s2t_transformer_m | 71M | 3.2 | 8.0 | 3.4 | 7.9 | [Download](https://dl.fbaipublicfiles.com/fairseq/s2t/librispeech_transformer_m.pt) | -| s2t_transformer_l | 268M | 3.0 | 7.5 | 3.2 | 7.5 | [Download](https://dl.fbaipublicfiles.com/fairseq/s2t/librispeech_transformer_l.pt) | - -[[Back]](..) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_text/docs/mtedx_example.md b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_text/docs/mtedx_example.md deleted file mode 100644 index 7e3d7595579363adf455bd5b55f5d0685de3a9e9..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_text/docs/mtedx_example.md +++ /dev/null @@ -1,201 +0,0 @@ -[[Back]](..) - -# S2T Example: Speech Translation (ST) on Multilingual TEDx - -[Multilingual TEDx](https://arxiv.org/abs/2102.01757) is multilingual corpus for speech recognition and -speech translation. The data is derived from TEDx talks in 8 source languages -with translations to a subset of 5 target languages. - -## Data Preparation -[Download](http://openslr.org/100/) and unpack Multilingual TEDx data to a path -`${MTEDX_ROOT}/${LANG_PAIR}`, then preprocess it with -```bash -# additional Python packages for S2T data processing/model training -pip install pandas torchaudio soundfile sentencepiece - -# Generate TSV manifests, features, vocabulary -# and configuration for each language -python examples/speech_to_text/prep_mtedx_data.py \ - --data-root ${MTEDX_ROOT} --task asr \ - --vocab-type unigram --vocab-size 1000 -python examples/speech_to_text/prep_mtedx_data.py \ - --data-root ${MTEDX_ROOT} --task st \ - --vocab-type unigram --vocab-size 1000 - -# Add vocabulary and configuration for joint data -# (based on the manifests and features generated above) -python examples/speech_to_text/prep_mtedx_data.py \ - --data-root ${MTEDX_ROOT} --task asr --joint \ - --vocab-type unigram --vocab-size 8000 -python examples/speech_to_text/prep_mtedx_data.py \ - --data-root ${MTEDX_ROOT} --task st --joint \ - --vocab-type unigram --vocab-size 8000 -``` -The generated files (manifest, features, vocabulary and data configuration) will be added to -`${MTEDX_ROOT}/${LANG_PAIR}` (per-language data) and `MTEDX_ROOT` (joint data). - - -## ASR -#### Training -Spanish as example: -```bash -fairseq-train ${MTEDX_ROOT}/es-es \ - --config-yaml config_asr.yaml --train-subset train_asr --valid-subset valid_asr \ - --save-dir ${ASR_SAVE_DIR} --num-workers 4 --max-tokens 40000 --max-epoch 200 \ - --task speech_to_text --criterion label_smoothed_cross_entropy --report-accuracy \ - --arch s2t_transformer_xs --optimizer adam --lr 2e-3 --lr-scheduler inverse_sqrt \ - --warmup-updates 10000 --clip-norm 10.0 --seed 1 --dropout 0.3 --label-smoothing 0.1 \ - --load-pretrained-encoder-from ${PRETRAINED_ENCODER} \ - --skip-invalid-size-inputs-valid-test \ - --keep-last-epochs 10 --update-freq 8 --patience 10 -``` -For joint model (using ASR data from all 8 languages): -```bash -fairseq-train ${MTEDX_ROOT} \ - --config-yaml config_asr.yaml \ - --train-subset train_es-es_asr,train_fr-fr_asr,train_pt-pt_asr,train_it-it_asr,train_ru-ru_asr,train_el-el_asr,train_ar-ar_asr,train_de-de_asr \ - --valid-subset valid_es-es_asr,valid_fr-fr_asr,valid_pt-pt_asr,valid_it-it_asr,valid_ru-ru_asr,valid_el-el_asr,valid_ar-ar_asr,valid_de-de_asr \ - --save-dir ${MULTILINGUAL_ASR_SAVE_DIR} --num-workers 4 --max-tokens 40000 --max-epoch 200 \ - --task speech_to_text --criterion label_smoothed_cross_entropy --report-accuracy \ - --arch s2t_transformer_s --optimizer adam --lr 2e-3 --lr-scheduler inverse_sqrt \ - --warmup-updates 10000 --clip-norm 10.0 --seed 1 --dropout 0.3 --label-smoothing 0.1 \ - --skip-invalid-size-inputs-valid-test \ - --keep-last-epochs 10 --update-freq 8 --patience 10 \ - --ignore-prefix-size 1 -``` -where `MULTILINGUAL_ASR_SAVE_DIR` is the checkpoint root path. We set `--update-freq 8` to simulate 8 GPUs -with 1 GPU. You may want to update it accordingly when using more than 1 GPU. -For multilingual models, we prepend target language ID token as target BOS, which should be excluded from -the training loss via `--ignore-prefix-size 1`. - -#### Inference & Evaluation -```bash -CHECKPOINT_FILENAME=avg_last_10_checkpoint.pt -python scripts/average_checkpoints.py \ - --inputs ${ASR_SAVE_DIR} --num-epoch-checkpoints 10 \ - --output "${ASR_SAVE_DIR}/${CHECKPOINT_FILENAME}" - -fairseq-generate ${MTEDX_ROOT}/es-es \ - --config-yaml config_asr.yaml --gen-subset test --task speech_to_text \ - --path ${ASR_SAVE_DIR}/${CHECKPOINT_FILENAME} --max-tokens 50000 --beam 5 \ - --skip-invalid-size-inputs-valid-test \ - --scoring wer --wer-tokenizer 13a --wer-lowercase --wer-remove-punct --remove-bpe - -# For models trained on joint data -CHECKPOINT_FILENAME=avg_last_10_checkpoint.pt -python scripts/average_checkpoints.py \ - --inputs ${MULTILINGUAL_ASR_SAVE_DIR} --num-epoch-checkpoints 10 \ - --output "${MULTILINGUAL_ASR_SAVE_DIR}/${CHECKPOINT_FILENAME}" - -for LANG in es fr pt it ru el ar de; do - fairseq-generate ${MTEDX_ROOT} \ - --config-yaml config_asr.yaml --gen-subset test_${LANG}-${LANG}_asr --task speech_to_text \ - --prefix-size 1 --path ${MULTILINGUAL_ASR_SAVE_DIR}/${CHECKPOINT_FILENAME} \ - --max-tokens 40000 --beam 5 \ - --skip-invalid-size-inputs-valid-test \ - --scoring wer --wer-tokenizer 13a --wer-lowercase --wer-remove-punct --remove-bpe -done -``` -#### Results -| Data | --arch | Params | Es | Fr | Pt | It | Ru | El | Ar | De | -|--------------|--------------------|--------|------|------|------|------|------|-------|-------|-------| -| Monolingual | s2t_transformer_xs | 10M | 46.4 | 45.6 | 54.8 | 48.0 | 74.7 | 109.5 | 104.4 | 111.1 | - - -## ST -#### Training -Es-En as example: -```bash -fairseq-train ${MTEDX_ROOT}/es-en \ - --config-yaml config_st.yaml --train-subset train_st --valid-subset valid_st \ - --save-dir ${ST_SAVE_DIR} --num-workers 4 --max-tokens 40000 --max-epoch 200 \ - --task speech_to_text --criterion label_smoothed_cross_entropy --report-accuracy \ - --arch s2t_transformer_xs --optimizer adam --lr 2e-3 --lr-scheduler inverse_sqrt \ - --warmup-updates 10000 --clip-norm 10.0 --seed 1 --dropout 0.3 --label-smoothing 0.1 \ - --load-pretrained-encoder-from ${PRETRAINED_ENCODER} \ - --skip-invalid-size-inputs-valid-test \ - --keep-last-epochs 10 --update-freq 8 --patience 10 -``` -For multilingual model (all 12 directions): -```bash -fairseq-train ${MTEDX_ROOT} \ - --config-yaml config_st.yaml \ - --train-subset train_el-en_st,train_es-en_st,train_es-fr_st,train_es-it_st,train_es-pt_st,train_fr-en_st,train_fr-es_st,train_fr-pt_st,train_it-en_st,train_it-es_st,train_pt-en_st,train_pt-es_st,train_ru-en_st \ - --valid-subset valid_el-en_st,valid_es-en_st,valid_es-fr_st,valid_es-it_st,valid_es-pt_st,valid_fr-en_st,valid_fr-es_st,valid_fr-pt_st,valid_it-en_st,valid_it-es_st,valid_pt-en_st,valid_pt-es_st,valid_ru-en_st \ - --save-dir ${MULTILINGUAL_ST_SAVE_DIR} --num-workers 4 --max-tokens 40000 --max-epoch 200 \ - --task speech_to_text --criterion label_smoothed_cross_entropy --report-accuracy \ - --arch s2t_transformer_s --optimizer adam --lr 2e-3 --lr-scheduler inverse_sqrt \ - --warmup-updates 10000 --clip-norm 10.0 --seed 1 --dropout 0.3 --label-smoothing 0.1 \ - --skip-invalid-size-inputs-valid-test \ - --keep-last-epochs 10 --update-freq 8 --patience 10 \ - --ignore-prefix-size 1 \ - --load-pretrained-encoder-from ${PRETRAINED_ENCODER} -``` -where `ST_SAVE_DIR` (`MULTILINGUAL_ST_SAVE_DIR`) is the checkpoint root path. The ST encoder is pre-trained by ASR -for faster training and better performance: `--load-pretrained-encoder-from <(JOINT_)ASR checkpoint path>`. We set -`--update-freq 8` to simulate 8 GPUs with 1 GPU. You may want to update it accordingly when using more than 1 GPU. -For multilingual models, we prepend target language ID token as target BOS, which should be excluded from -the training loss via `--ignore-prefix-size 1`. - -#### Inference & Evaluation -Average the last 10 checkpoints and evaluate on the `test` split: -```bash -CHECKPOINT_FILENAME=avg_last_10_checkpoint.pt -python scripts/average_checkpoints.py \ - --inputs ${ST_SAVE_DIR} --num-epoch-checkpoints 10 \ - --output "${ST_SAVE_DIR}/${CHECKPOINT_FILENAME}" - -fairseq-generate ${MTEDX_ROOT}/es-en \ - --config-yaml config_st.yaml --gen-subset test --task speech_to_text \ - --path ${ST_SAVE_DIR}/${CHECKPOINT_FILENAME} \ - --max-tokens 50000 --beam 5 --scoring sacrebleu --remove-bpe - -# For multilingual models -python scripts/average_checkpoints.py \ - --inputs ${MULTILINGUAL_ST_SAVE_DIR} --num-epoch-checkpoints 10 \ - --output "${MULTILINGUAL_ST_SAVE_DIR}/${CHECKPOINT_FILENAME}" - -for LANGPAIR in es-en es-fr es-pt fr-en fr-es fr-pt pt-en pt-es it-en it-es ru-en el-en; do - fairseq-generate ${MTEDX_ROOT} \ - --config-yaml config_st.yaml --gen-subset test_${LANGPAIR}_st --task speech_to_text \ - --prefix-size 1 --path ${MULTILINGUAL_ST_SAVE_DIR}/${CHECKPOINT_FILENAME} \ - --max-tokens 40000 --beam 5 \ - --skip-invalid-size-inputs-valid-test \ - --scoring sacrebleu --remove-bpe -done -``` -For multilingual models, we force decoding from the target language ID token (as BOS) via `--prefix-size 1`. - -#### Results -| Data | --arch | Params | Es-En | Es-Pt | Es-Fr | Fr-En | Fr-Es | Fr-Pt | Pt-En | Pt-Es | It-En | It-Es | Ru-En | El-En | -|--------------|--------------------|-----|-------|-------|-------|-------|-------|-------|-------|-------|-------|-------|-------|-------| -| Bilingual | s2t_transformer_xs | 10M | 7.0 | 12.2 | 1.7 | 8.9 | 10.6 | 7.9 | 8.1 | 8.7 | 6.4 | 1.0 | 0.7 | 0.6 | -| Multilingual | s2t_transformer_s | 31M | 12.3 | 17.4 | 6.1 | 12.0 | 13.6 | 13.2 | 12.0 | 13.7 | 10.7 | 13.1 | 0.6 | 0.8 | - - -## Citation -Please cite as: -``` -@inproceedings{salesky2021mtedx, - title={Multilingual TEDx Corpus for Speech Recognition and Translation}, - author={Elizabeth Salesky and Matthew Wiesner and Jacob Bremerman and Roldano Cattoni and Matteo Negri and Marco Turchi and Douglas W. Oard and Matt Post}, - booktitle={Proceedings of Interspeech}, - year={2021}, -} - -@inproceedings{wang2020fairseqs2t, - title = {fairseq S2T: Fast Speech-to-Text Modeling with fairseq}, - author = {Changhan Wang and Yun Tang and Xutai Ma and Anne Wu and Dmytro Okhonko and Juan Pino}, - booktitle = {Proceedings of the 2020 Conference of the Asian Chapter of the Association for Computational Linguistics (AACL): System Demonstrations}, - year = {2020}, -} - -@inproceedings{ott2019fairseq, - title = {fairseq: A Fast, Extensible Toolkit for Sequence Modeling}, - author = {Myle Ott and Sergey Edunov and Alexei Baevski and Angela Fan and Sam Gross and Nathan Ng and David Grangier and Michael Auli}, - booktitle = {Proceedings of NAACL-HLT 2019: Demonstrations}, - year = {2019}, -} -``` - -[[Back]](..) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_text/docs/mustc_example.md b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_text/docs/mustc_example.md deleted file mode 100644 index c95ef3e15660107c3384f87c1680f005044e7f3b..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_text/docs/mustc_example.md +++ /dev/null @@ -1,155 +0,0 @@ -[[Back]](..) - -# S2T Example: Speech Translation (ST) on MuST-C - -[MuST-C](https://www.aclweb.org/anthology/N19-1202) is multilingual speech-to-text translation corpus with -8-language translations on English TED talks. We match the state-of-the-art performance in -[ESPNet-ST](https://arxiv.org/pdf/2004.10234.pdf) with a simpler model training pipeline. - -## Data Preparation -[Download](https://ict.fbk.eu/must-c) and unpack MuST-C data to a path -`${MUSTC_ROOT}/en-${TARGET_LANG_ID}`, then preprocess it with -```bash -# additional Python packages for S2T data processing/model training -pip install pandas torchaudio soundfile sentencepiece - -# Generate TSV manifests, features, vocabulary -# and configuration for each language -python examples/speech_to_text/prep_mustc_data.py \ - --data-root ${MUSTC_ROOT} --task asr \ - --vocab-type unigram --vocab-size 5000 -python examples/speech_to_text/prep_mustc_data.py \ - --data-root ${MUSTC_ROOT} --task st \ - --vocab-type unigram --vocab-size 8000 - -# Add vocabulary and configuration for joint data -# (based on the manifests and features generated above) -python examples/speech_to_text/prep_mustc_data.py \ - --data-root ${MUSTC_ROOT} --task asr --joint \ - --vocab-type unigram --vocab-size 10000 -python examples/speech_to_text/prep_mustc_data.py \ - --data-root ${MUSTC_ROOT} --task st --joint \ - --vocab-type unigram --vocab-size 10000 -``` -The generated files (manifest, features, vocabulary and data configuration) will be added to -`${MUSTC_ROOT}/en-${TARGET_LANG_ID}` (per-language data) and `MUSTC_ROOT` (joint data). - -Download our vocabulary files if you want to use our pre-trained models: -- ASR: [En-De](https://dl.fbaipublicfiles.com/fairseq/s2t/mustc_de_asr_vocab_unigram5000.zip), [En-Nl](https://dl.fbaipublicfiles.com/fairseq/s2t/mustc_nl_asr_vocab_unigram5000.zip), [En-Es](https://dl.fbaipublicfiles.com/fairseq/s2t/mustc_es_asr_vocab_unigram5000.zip), [En-Fr](https://dl.fbaipublicfiles.com/fairseq/s2t/mustc_fr_asr_vocab_unigram5000.zip), [En-It](https://dl.fbaipublicfiles.com/fairseq/s2t/mustc_it_asr_vocab_unigram5000.zip), [En-Pt](https://dl.fbaipublicfiles.com/fairseq/s2t/mustc_pt_asr_vocab_unigram5000.zip), [En-Ro](https://dl.fbaipublicfiles.com/fairseq/s2t/mustc_ro_asr_vocab_unigram5000.zip), [En-Ru](https://dl.fbaipublicfiles.com/fairseq/s2t/mustc_ru_asr_vocab_unigram5000.zip), [Joint](https://dl.fbaipublicfiles.com/fairseq/s2t/mustc_joint_asr_vocab_unigram10000.zip) -- ST: [En-De](https://dl.fbaipublicfiles.com/fairseq/s2t/mustc_de_st_vocab_unigram8000.zip), [En-Nl](https://dl.fbaipublicfiles.com/fairseq/s2t/mustc_nl_st_vocab_unigram8000.zip), [En-Es](https://dl.fbaipublicfiles.com/fairseq/s2t/mustc_es_st_vocab_unigram8000.zip), [En-Fr](https://dl.fbaipublicfiles.com/fairseq/s2t/mustc_fr_st_vocab_unigram8000.zip), [En-It](https://dl.fbaipublicfiles.com/fairseq/s2t/mustc_it_st_vocab_unigram8000.zip), [En-Pt](https://dl.fbaipublicfiles.com/fairseq/s2t/mustc_pt_st_vocab_unigram8000.zip), [En-Ro](https://dl.fbaipublicfiles.com/fairseq/s2t/mustc_ro_st_vocab_unigram8000.zip), [En-Ru](https://dl.fbaipublicfiles.com/fairseq/s2t/mustc_ru_st_vocab_unigram8000.zip), [Multilingual](https://dl.fbaipublicfiles.com/fairseq/s2t/mustc_multilingual_st_vocab_unigram10000.zip) - -## ASR -#### Training -En-De as example: -```bash -fairseq-train ${MUSTC_ROOT}/en-de \ - --config-yaml config_asr.yaml --train-subset train_asr --valid-subset dev_asr \ - --save-dir ${ASR_SAVE_DIR} --num-workers 4 --max-tokens 40000 --max-update 100000 \ - --task speech_to_text --criterion label_smoothed_cross_entropy --label-smoothing 0.1 --report-accuracy \ - --arch s2t_transformer_s --optimizer adam --lr 1e-3 --lr-scheduler inverse_sqrt \ - --warmup-updates 10000 --clip-norm 10.0 --seed 1 --update-freq 8 -``` -For joint model (using ASR data from all 8 directions): -```bash -fairseq-train ${MUSTC_ROOT} \ - --config-yaml config_asr.yaml \ - --train-subset train_de_asr,train_nl_asr,train_es_asr,train_fr_asr,train_it_asr,train_pt_asr,train_ro_asr,train_ru_asr \ - --valid-subset dev_de_asr,dev_nl_asr,dev_es_asr,dev_fr_asr,dev_it_asr,dev_pt_asr,dev_ro_asr,dev_ru_asr \ - --save-dir ${JOINT_ASR_SAVE_DIR} --num-workers 4 --max-tokens 40000 --max-update 100000 \ - --task speech_to_text --criterion label_smoothed_cross_entropy --label-smoothing 0.1 --report-accuracy \ - --arch s2t_transformer_s --optimizer adam --lr 1e-3 --lr-scheduler inverse_sqrt \ - --warmup-updates 10000 --clip-norm 10.0 --seed 1 --update-freq 8 -``` -where `ASR_SAVE_DIR` (`JOINT_ASR_SAVE_DIR`) is the checkpoint root path. We set `--update-freq 8` to simulate 8 GPUs -with 1 GPU. You may want to update it accordingly when using more than 1 GPU. - -#### Inference & Evaluation -```bash -CHECKPOINT_FILENAME=avg_last_10_checkpoint.pt -python scripts/average_checkpoints.py \ - --inputs ${ASR_SAVE_DIR} --num-epoch-checkpoints 10 \ - --output "${ASR_SAVE_DIR}/${CHECKPOINT_FILENAME}" -fairseq-generate ${MUSTC_ROOT}/en-de \ - --config-yaml config_asr.yaml --gen-subset tst-COMMON_asr --task speech_to_text \ - --path ${ASR_SAVE_DIR}/${CHECKPOINT_FILENAME} --max-tokens 50000 --beam 5 \ - --scoring wer --wer-tokenizer 13a --wer-lowercase --wer-remove-punct - -# For models trained on joint data -python scripts/average_checkpoints.py \ - --inputs ${JOINT_ASR_SAVE_DIR} --num-epoch-checkpoints 10 \ - --output "${JOINT_ASR_SAVE_DIR}/${CHECKPOINT_FILENAME}" -for LANG in de nl es fr it pt ro ru; do - fairseq-generate ${MUSTC_ROOT} \ - --config-yaml config_asr.yaml --gen-subset tst-COMMON_${LANG}_asr --task speech_to_text \ - --path ${JOINT_ASR_SAVE_DIR}/${CHECKPOINT_FILENAME} --max-tokens 50000 --beam 5 \ - --scoring wer --wer-tokenizer 13a --wer-lowercase --wer-remove-punct -done -``` -#### Results -| Data | --arch | Params | En-De | En-Nl | En-Es | En-Fr | En-It | En-Pt | En-Ro | En-Ru | Model | -|---|---|---|---|---|---|---|---|---|---|---|---| -| Single | s2t_transformer_s | 31M | [18.2](https://dl.fbaipublicfiles.com/fairseq/s2t/mustc_de_asr_transformer_s.pt) | [17.6](https://dl.fbaipublicfiles.com/fairseq/s2t/mustc_nl_asr_transformer_s.pt) | [17.7](https://dl.fbaipublicfiles.com/fairseq/s2t/mustc_es_asr_transformer_s.pt) | [17.2](https://dl.fbaipublicfiles.com/fairseq/s2t/mustc_fr_asr_transformer_s.pt) | [17.9](https://dl.fbaipublicfiles.com/fairseq/s2t/mustc_it_asr_transformer_s.pt) | [19.1](https://dl.fbaipublicfiles.com/fairseq/s2t/mustc_pt_asr_transformer_s.pt) | [18.1](https://dl.fbaipublicfiles.com/fairseq/s2t/mustc_ro_asr_transformer_s.pt) | [17.7](https://dl.fbaipublicfiles.com/fairseq/s2t/mustc_ru_asr_transformer_s.pt) | (<-Download) | -| Joint | s2t_transformer_m | 76M | 16.8 | 16.7 | 16.9 | 16.9 | 17.0 | 17.4 | 17.0 | 16.9 | [Download](https://dl.fbaipublicfiles.com/fairseq/s2t/mustc_joint_asr_transformer_m.pt) | - -## ST -#### Training -En-De as example: -```bash -fairseq-train ${MUSTC_ROOT}/en-de \ - --config-yaml config_st.yaml --train-subset train_st --valid-subset dev_st \ - --save-dir ${ST_SAVE_DIR} --num-workers 4 --max-tokens 40000 --max-update 100000 \ - --task speech_to_text --criterion label_smoothed_cross_entropy --label-smoothing 0.1 --report-accuracy \ - --arch s2t_transformer_s --optimizer adam --lr 2e-3 --lr-scheduler inverse_sqrt \ - --warmup-updates 10000 --clip-norm 10.0 --seed 1 --update-freq 8 \ - --load-pretrained-encoder-from ${ASR_SAVE_DIR}/${CHECKPOINT_FILENAME} -``` -For multilingual model (all 8 directions): -```bash -fairseq-train ${MUSTC_ROOT} \ - --config-yaml config_st.yaml \ - --train-subset train_de_st,train_nl_st,train_es_st,train_fr_st,train_it_st,train_pt_st,train_ro_st,train_ru_st \ - --valid-subset dev_de_st,dev_nl_st,dev_es_st,dev_fr_st,dev_it_st,dev_pt_st,dev_ro_st,dev_ru_st \ - --save-dir ${MULTILINGUAL_ST_SAVE_DIR} --num-workers 4 --max-tokens 40000 --max-update 100000 \ - --task speech_to_text --criterion label_smoothed_cross_entropy --label-smoothing 0.1 --report-accuracy \ - --arch s2t_transformer_s --ignore-prefix-size 1 --optimizer adam --lr 2e-3 --lr-scheduler inverse_sqrt \ - --warmup-updates 10000 --clip-norm 10.0 --seed 1 --update-freq 8 \ - --load-pretrained-encoder-from ${JOINT_ASR_SAVE_DIR}/${CHECKPOINT_FILENAME} -``` -where `ST_SAVE_DIR` (`MULTILINGUAL_ST_SAVE_DIR`) is the checkpoint root path. The ST encoder is pre-trained by ASR -for faster training and better performance: `--load-pretrained-encoder-from <(JOINT_)ASR checkpoint path>`. We set -`--update-freq 8` to simulate 8 GPUs with 1 GPU. You may want to update it accordingly when using more than 1 GPU. -For multilingual models, we prepend target language ID token as target BOS, which should be excluded from -the training loss via `--ignore-prefix-size 1`. - -#### Inference & Evaluation -Average the last 10 checkpoints and evaluate on the `tst-COMMON` split: -```bash -CHECKPOINT_FILENAME=avg_last_10_checkpoint.pt -python scripts/average_checkpoints.py \ - --inputs ${ST_SAVE_DIR} --num-epoch-checkpoints 10 \ - --output "${ST_SAVE_DIR}/${CHECKPOINT_FILENAME}" -fairseq-generate ${MUSTC_ROOT}/en-de \ - --config-yaml config_st.yaml --gen-subset tst-COMMON_st --task speech_to_text \ - --path ${ST_SAVE_DIR}/${CHECKPOINT_FILENAME} \ - --max-tokens 50000 --beam 5 --scoring sacrebleu - -# For multilingual models -python scripts/average_checkpoints.py \ - --inputs ${MULTILINGUAL_ST_SAVE_DIR} --num-epoch-checkpoints 10 \ - --output "${MULTILINGUAL_ST_SAVE_DIR}/${CHECKPOINT_FILENAME}" -for LANG in de nl es fr it pt ro ru; do - fairseq-generate ${MUSTC_ROOT} \ - --config-yaml config_st.yaml --gen-subset tst-COMMON_${LANG}_st --task speech_to_text \ - --prefix-size 1 --path ${MULTILINGUAL_ST_SAVE_DIR}/${CHECKPOINT_FILENAME} \ - --max-tokens 50000 --beam 5 --scoring sacrebleu -done -``` -For multilingual models, we force decoding from the target language ID token (as BOS) via `--prefix-size 1`. - -#### Results -| Data | --arch | Params | En-De | En-Nl | En-Es | En-Fr | En-It | En-Pt | En-Ro | En-Ru | Model | -|---|---|---|---|---|---|---|---|---|---|---|---| -| Bilingual | s2t_transformer_s | 31M | [22.7](https://dl.fbaipublicfiles.com/fairseq/s2t/mustc_de_st_transformer_s.pt) | [27.3](https://dl.fbaipublicfiles.com/fairseq/s2t/mustc_nl_st_transformer_s.pt) | [27.2](https://dl.fbaipublicfiles.com/fairseq/s2t/mustc_es_st_transformer_s.pt) | [32.9](https://dl.fbaipublicfiles.com/fairseq/s2t/mustc_fr_st_transformer_s.pt) | [22.7](https://dl.fbaipublicfiles.com/fairseq/s2t/mustc_it_st_transformer_s.pt) | [28.1](https://dl.fbaipublicfiles.com/fairseq/s2t/mustc_pt_st_transformer_s.pt) | [21.9](https://dl.fbaipublicfiles.com/fairseq/s2t/mustc_ro_st_transformer_s.pt) | [15.3](https://dl.fbaipublicfiles.com/fairseq/s2t/mustc_ru_st_transformer_s.pt) | (<-Download) | -| Multilingual | s2t_transformer_m | 76M | 24.5 | 28.6 | 28.2 | 34.9 | 24.6 | 31.1 | 23.8 | 16.0 | [Download](https://dl.fbaipublicfiles.com/fairseq/s2t/mustc_multilingual_st_transformer_m.pt) | - -[[Back]](..) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_text/docs/simulst_mustc_example.md b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_text/docs/simulst_mustc_example.md deleted file mode 100644 index f3b5a413a27bbe2700da3f418460aa0a7c41abdd..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_text/docs/simulst_mustc_example.md +++ /dev/null @@ -1,190 +0,0 @@ -# Simultaneous Speech Translation (SimulST) on MuST-C - -This is a tutorial of training and evaluating a transformer *wait-k* simultaneous model on MUST-C English-Germen Dataset, from [SimulMT to SimulST: Adapting Simultaneous Text Translation to End-to-End Simultaneous Speech Translation](https://www.aclweb.org/anthology/2020.aacl-main.58.pdf). - -[MuST-C](https://www.aclweb.org/anthology/N19-1202) is multilingual speech-to-text translation corpus with 8-language translations on English TED talks. - -## Data Preparation -This section introduces the data preparation for training and evaluation. -If you only want to evaluate the model, please jump to [Inference & Evaluation](#inference--evaluation) - -[Download](https://ict.fbk.eu/must-c) and unpack MuST-C data to a path -`${MUSTC_ROOT}/en-${TARGET_LANG_ID}`, then preprocess it with -```bash -# Additional Python packages for S2T data processing/model training -pip install pandas torchaudio sentencepiece - -# Generate TSV manifests, features, vocabulary, -# global cepstral and mean estimation, -# and configuration for each language -cd fairseq - -python examples/speech_to_text/prep_mustc_data.py \ - --data-root ${MUSTC_ROOT} --task asr \ - --vocab-type unigram --vocab-size 10000 \ - --cmvn-type global - -python examples/speech_to_text/prep_mustc_data.py \ - --data-root ${MUSTC_ROOT} --task st \ - --vocab-type unigram --vocab-size 10000 \ - --cmvn-type global -``` - -## ASR Pretraining -We need a pretrained offline ASR model. Assuming the save directory of the ASR model is `${ASR_SAVE_DIR}`. -The following command (and the subsequent training commands in this tutorial) assume training on 1 GPU (you can also train on 8 GPUs and remove the `--update-freq 8` option). -``` -fairseq-train ${MUSTC_ROOT}/en-de \ - --config-yaml config_asr.yaml --train-subset train_asr --valid-subset dev_asr \ - --save-dir ${ASR_SAVE_DIR} --num-workers 4 --max-tokens 40000 --max-update 100000 \ - --task speech_to_text --criterion label_smoothed_cross_entropy --report-accuracy \ - --arch convtransformer_espnet --optimizer adam --lr 0.0005 --lr-scheduler inverse_sqrt \ - --warmup-updates 10000 --clip-norm 10.0 --seed 1 --update-freq 8 -``` -A pretrained ASR checkpoint can be downloaded [here](https://dl.fbaipublicfiles.com/simultaneous_translation/must_c_v1_en_de_pretrained_asr) - -## Simultaneous Speech Translation Training - -### Wait-K with fixed pre-decision module -Fixed pre-decision indicates that the model operate simultaneous policy on the boundaries of fixed chunks. -Here is a example of fixed pre-decision ratio 7 (the simultaneous decision is made every 7 encoder states) and -a wait-3 policy model. Assuming the save directory is `${ST_SAVE_DIR}` -```bash - fairseq-train ${MUSTC_ROOT}/en-de \ - --config-yaml config_st.yaml --train-subset train_st --valid-subset dev_st \ - --save-dir ${ST_SAVE_DIR} --num-workers 8 \ - --optimizer adam --lr 0.0001 --lr-scheduler inverse_sqrt --clip-norm 10.0 \ - --criterion label_smoothed_cross_entropy \ - --warmup-updates 4000 --max-update 100000 --max-tokens 40000 --seed 2 \ - --load-pretrained-encoder-from ${ASR_SAVE_DIR}/checkpoint_best.pt \ - --task speech_to_text \ - --arch convtransformer_simul_trans_espnet \ - --simul-type waitk_fixed_pre_decision \ - --waitk-lagging 3 \ - --fixed-pre-decision-ratio 7 \ - --update-freq 8 - -``` -### Monotonic multihead attention with fixed pre-decision module -``` - fairseq-train ${MUSTC_ROOT}/en-de \ - --config-yaml config_st.yaml --train-subset train_st --valid-subset dev_st \ - --save-dir ${ST_SAVE_DIR} --num-workers 8 \ - --optimizer adam --lr 0.0001 --lr-scheduler inverse_sqrt --clip-norm 10.0 \ - --warmup-updates 4000 --max-update 100000 --max-tokens 40000 --seed 2 \ - --load-pretrained-encoder-from ${ASR_SAVE_DIR}/${CHECKPOINT_FILENAME} \ - --task speech_to_text \ - --criterion latency_augmented_label_smoothed_cross_entropy \ - --latency-weight-avg 0.1 \ - --arch convtransformer_simul_trans_espnet \ - --simul-type infinite_lookback_fixed_pre_decision \ - --fixed-pre-decision-ratio 7 \ - --update-freq 8 -``` -## Inference & Evaluation -[SimulEval](https://github.com/facebookresearch/SimulEval) is used for evaluation. -The following command is for evaluation. - -``` -git clone https://github.com/facebookresearch/SimulEval.git -cd SimulEval -pip install -e . - -simuleval \ - --agent ${FAIRSEQ}/examples/speech_to_text/simultaneous_translation/agents/fairseq_simul_st_agent.py - --source ${SRC_LIST_OF_AUDIO} - --target ${TGT_FILE} - --data-bin ${MUSTC_ROOT}/en-de \ - --config config_st.yaml \ - --model-path ${ST_SAVE_DIR}/${CHECKPOINT_FILENAME} \ - --output ${OUTPUT} \ - --scores -``` - -The source file `${SRC_LIST_OF_AUDIO}` is a list of paths of audio files. Assuming your audio files stored at `/home/user/data`, -it should look like this - -```bash -/home/user/data/audio-1.wav -/home/user/data/audio-2.wav -``` - -Each line of target file `${TGT_FILE}` is the translation for each audio file input. -```bash -Translation_1 -Translation_2 -``` -The evaluation runs on the original MUSTC segmentation. -The following command will generate the wav list and text file for a evaluation set `${SPLIT}` (chose from `dev`, `tst-COMMON` and `tst-HE`) in MUSTC to `${EVAL_DATA}`. -```bash -python ${FAIRSEQ}/examples/speech_to_text/seg_mustc_data.py \ - --data-root ${MUSTC_ROOT} --lang de \ - --split ${SPLIT} --task st \ - --output ${EVAL_DATA} -``` - -The `--data-bin` and `--config` should be the same in previous section if you prepare the data from the scratch. -If only for evaluation, a prepared data directory can be found [here](https://dl.fbaipublicfiles.com/simultaneous_translation/must_c_v1.0_en_de_databin.tgz). It contains -- `spm_unigram10000_st.model`: a sentencepiece model binary. -- `spm_unigram10000_st.txt`: the dictionary file generated by the sentencepiece model. -- `gcmvn.npz`: the binary for global cepstral mean and variance. -- `config_st.yaml`: the config yaml file. It looks like this. -You will need to set the absolute paths for `sentencepiece_model` and `stats_npz_path` if the data directory is downloaded. -```yaml -bpe_tokenizer: - bpe: sentencepiece - sentencepiece_model: ABS_PATH_TO_SENTENCEPIECE_MODEL -global_cmvn: - stats_npz_path: ABS_PATH_TO_GCMVN_FILE -input_channels: 1 -input_feat_per_channel: 80 -sampling_alpha: 1.0 -specaugment: - freq_mask_F: 27 - freq_mask_N: 1 - time_mask_N: 1 - time_mask_T: 100 - time_mask_p: 1.0 - time_wrap_W: 0 -transforms: - '*': - - global_cmvn - _train: - - global_cmvn - - specaugment -vocab_filename: spm_unigram10000_st.txt -``` - -Notice that once a `--data-bin` is set, the `--config` is the base name of the config yaml, not the full path. - -Set `--model-path` to the model checkpoint. -A pretrained checkpoint can be downloaded from [here](https://dl.fbaipublicfiles.com/simultaneous_translation/convtransformer_wait5_pre7), which is a wait-5 model with a pre-decision of 280 ms. - -The result of this model on `tst-COMMON` is: -```bash -{ - "Quality": { - "BLEU": 13.94974229366959 - }, - "Latency": { - "AL": 1751.8031870037803, - "AL_CA": 2338.5911762796536, - "AP": 0.7931395378788959, - "AP_CA": 0.9405103863210942, - "DAL": 1987.7811616943081, - "DAL_CA": 2425.2751560926167 - } -} -``` - -If `--output ${OUTPUT}` option is used, the detailed log and scores will be stored under the `${OUTPUT}` directory. - - -The quality is measured by detokenized BLEU. So make sure that the predicted words sent to the server are detokenized. - -The latency metrics are -* Average Proportion -* Average Lagging -* Differentiable Average Lagging - -Again they will also be evaluated on detokenized text. diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_text/prep_covost_data.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_text/prep_covost_data.py deleted file mode 100644 index e4f491befd0bf67c387b660fce419f89dd8e5c7a..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_text/prep_covost_data.py +++ /dev/null @@ -1,278 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import logging -import shutil -from pathlib import Path -from tempfile import NamedTemporaryFile -from typing import Optional, Tuple - -import pandas as pd -import torchaudio -from examples.speech_to_text.data_utils import ( - create_zip, - extract_fbank_features, - filter_manifest_df, - gen_config_yaml, - gen_vocab, - get_zip_manifest, - load_df_from_tsv, - save_df_to_tsv, -) -from torch import Tensor -from torch.utils.data import Dataset -from torchaudio.datasets.utils import download_url, extract_archive -from tqdm import tqdm - -log = logging.getLogger(__name__) - - -MANIFEST_COLUMNS = ["id", "audio", "n_frames", "tgt_text", "speaker"] - - -class CoVoST(Dataset): - """Create a Dataset for CoVoST (https://github.com/facebookresearch/covost). - - Args: - root (str): root path to the dataset and generated manifests/features - source_language (str): source (audio) language - target_language (str, optional): target (text) language, - None for no translation (default: None) - version (int, optional): CoVoST version. (default: 2) - download (bool, optional): Whether to download the dataset if it is not - found at root path. (default: ``False``). - """ - - COVOST_URL_TEMPLATE = ( - "https://dl.fbaipublicfiles.com/covost/" - "covost_v2.{src_lang}_{tgt_lang}.tsv.tar.gz" - ) - - VERSIONS = {2} - SPLITS = ["train", "dev", "test"] - - XX_EN_LANGUAGES = { - 1: ["fr", "de", "nl", "ru", "es", "it", "tr", "fa", "sv-SE", "mn", "zh-CN"], - 2: [ - "fr", - "de", - "es", - "ca", - "it", - "ru", - "zh-CN", - "pt", - "fa", - "et", - "mn", - "nl", - "tr", - "ar", - "sv-SE", - "lv", - "sl", - "ta", - "ja", - "id", - "cy", - ], - } - EN_XX_LANGUAGES = { - 1: [], - 2: [ - "de", - "tr", - "fa", - "sv-SE", - "mn", - "zh-CN", - "cy", - "ca", - "sl", - "et", - "id", - "ar", - "ta", - "lv", - "ja", - ], - } - - def __init__( - self, - root: str, - split: str, - source_language: str, - target_language: Optional[str] = None, - version: int = 2, - ) -> None: - assert version in self.VERSIONS and split in self.SPLITS - assert source_language is not None - self.no_translation = target_language is None - if not self.no_translation: - assert "en" in {source_language, target_language} - if source_language == "en": - assert target_language in self.EN_XX_LANGUAGES[version] - else: - assert source_language in self.XX_EN_LANGUAGES[version] - else: - # Hack here so that we can get "split" column from CoVoST TSV. - # Note that we use CoVoST train split for ASR which is an extension - # to Common Voice train split. - target_language = "de" if source_language == "en" else "en" - - self.root: Path = Path(root) - - cv_tsv_path = self.root / "validated.tsv" - assert cv_tsv_path.is_file() - - covost_url = self.COVOST_URL_TEMPLATE.format( - src_lang=source_language, tgt_lang=target_language - ) - covost_archive = self.root / Path(covost_url).name - if not covost_archive.is_file(): - download_url(covost_url, self.root.as_posix(), hash_value=None) - extract_archive(covost_archive.as_posix()) - - cv_tsv = load_df_from_tsv(cv_tsv_path) - covost_tsv = load_df_from_tsv( - self.root / Path(covost_url).name.replace(".tar.gz", "") - ) - df = pd.merge( - left=cv_tsv[["path", "sentence", "client_id"]], - right=covost_tsv[["path", "translation", "split"]], - how="inner", - on="path", - ) - if split == "train": - df = df[(df["split"] == split) | (df["split"] == f"{split}_covost")] - else: - df = df[df["split"] == split] - data = df.to_dict(orient="index").items() - data = [v for k, v in sorted(data, key=lambda x: x[0])] - self.data = [] - for e in data: - try: - path = self.root / "clips" / e["path"] - _ = torchaudio.info(path.as_posix()) - self.data.append(e) - except RuntimeError: - pass - - def __getitem__( - self, n: int - ) -> Tuple[Tensor, int, str, str, Optional[str], str, str]: - """Load the n-th sample from the dataset. - - Args: - n (int): The index of the sample to be loaded - - Returns: - tuple: ``(waveform, sample_rate, sentence, translation, speaker_id, - sample_id)`` - """ - data = self.data[n] - path = self.root / "clips" / data["path"] - waveform, sample_rate = torchaudio.load(path) - sentence = data["sentence"] - translation = None if self.no_translation else data["translation"] - speaker_id = data["client_id"] - _id = data["path"].replace(".mp3", "") - return waveform, sample_rate, sentence, translation, speaker_id, _id - - def __len__(self) -> int: - return len(self.data) - - -def process(args): - root = Path(args.data_root).absolute() / args.src_lang - if not root.is_dir(): - raise NotADirectoryError(f"{root} does not exist") - # Extract features - feature_root = root / "fbank80" - feature_root.mkdir(exist_ok=True) - for split in CoVoST.SPLITS: - print(f"Fetching split {split}...") - dataset = CoVoST(root, split, args.src_lang, args.tgt_lang) - print("Extracting log mel filter bank features...") - for waveform, sample_rate, _, _, _, utt_id in tqdm(dataset): - extract_fbank_features( - waveform, sample_rate, feature_root / f"{utt_id}.npy" - ) - # Pack features into ZIP - zip_path = root / "fbank80.zip" - print("ZIPing features...") - create_zip(feature_root, zip_path) - print("Fetching ZIP manifest...") - audio_paths, audio_lengths = get_zip_manifest(zip_path) - # Generate TSV manifest - print("Generating manifest...") - train_text = [] - task = f"asr_{args.src_lang}" - if args.tgt_lang is not None: - task = f"st_{args.src_lang}_{args.tgt_lang}" - for split in CoVoST.SPLITS: - manifest = {c: [] for c in MANIFEST_COLUMNS} - dataset = CoVoST(root, split, args.src_lang, args.tgt_lang) - for _, _, src_utt, tgt_utt, speaker_id, utt_id in tqdm(dataset): - manifest["id"].append(utt_id) - manifest["audio"].append(audio_paths[utt_id]) - manifest["n_frames"].append(audio_lengths[utt_id]) - manifest["tgt_text"].append(src_utt if args.tgt_lang is None else tgt_utt) - manifest["speaker"].append(speaker_id) - is_train_split = split.startswith("train") - if is_train_split: - train_text.extend(manifest["tgt_text"]) - df = pd.DataFrame.from_dict(manifest) - df = filter_manifest_df(df, is_train_split=is_train_split) - save_df_to_tsv(df, root / f"{split}_{task}.tsv") - # Generate vocab - vocab_size_str = "" if args.vocab_type == "char" else str(args.vocab_size) - spm_filename_prefix = f"spm_{args.vocab_type}{vocab_size_str}_{task}" - with NamedTemporaryFile(mode="w") as f: - for t in train_text: - f.write(t + "\n") - gen_vocab( - Path(f.name), - root / spm_filename_prefix, - args.vocab_type, - args.vocab_size - ) - # Generate config YAML - gen_config_yaml( - root, - spm_filename=spm_filename_prefix + ".model", - yaml_filename=f"config_{task}.yaml", - specaugment_policy="lb", - ) - # Clean up - shutil.rmtree(feature_root) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument( - "--data-root", "-d", required=True, type=str, - help="data root with sub-folders for each language /" - ) - parser.add_argument( - "--vocab-type", - default="unigram", - required=True, - type=str, - choices=["bpe", "unigram", "char"], - ), - parser.add_argument("--vocab-size", default=1000, type=int) - parser.add_argument("--src-lang", "-s", required=True, type=str) - parser.add_argument("--tgt-lang", "-t", type=str) - args = parser.parse_args() - - process(args) - - -if __name__ == "__main__": - main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_text/prep_librispeech_data.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_text/prep_librispeech_data.py deleted file mode 100644 index 60c15cf74a25de808b5993fe97ad51a7cbff05b9..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_text/prep_librispeech_data.py +++ /dev/null @@ -1,118 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import logging -import shutil -from pathlib import Path -from tempfile import NamedTemporaryFile - -import pandas as pd -from examples.speech_to_text.data_utils import ( - create_zip, - extract_fbank_features, - gen_config_yaml, - gen_vocab, - get_zip_manifest, - save_df_to_tsv, -) -from torchaudio.datasets import LIBRISPEECH -from tqdm import tqdm - -log = logging.getLogger(__name__) - -SPLITS = [ - "train-clean-100", - "train-clean-360", - "train-other-500", - "dev-clean", - "dev-other", - "test-clean", - "test-other", -] - -MANIFEST_COLUMNS = ["id", "audio", "n_frames", "tgt_text", "speaker"] - - -def process(args): - out_root = Path(args.output_root).absolute() - out_root.mkdir(exist_ok=True) - # Extract features - feature_root = out_root / "fbank80" - feature_root.mkdir(exist_ok=True) - for split in SPLITS: - print(f"Fetching split {split}...") - dataset = LIBRISPEECH(out_root.as_posix(), url=split, download=True) - print("Extracting log mel filter bank features...") - for wav, sample_rate, _, spk_id, chapter_no, utt_no in tqdm(dataset): - sample_id = f"{spk_id}-{chapter_no}-{utt_no}" - extract_fbank_features( - wav, sample_rate, feature_root / f"{sample_id}.npy" - ) - # Pack features into ZIP - zip_path = out_root / "fbank80.zip" - print("ZIPing features...") - create_zip(feature_root, zip_path) - print("Fetching ZIP manifest...") - audio_paths, audio_lengths = get_zip_manifest(zip_path) - # Generate TSV manifest - print("Generating manifest...") - train_text = [] - for split in SPLITS: - manifest = {c: [] for c in MANIFEST_COLUMNS} - dataset = LIBRISPEECH(out_root.as_posix(), url=split) - for _, _, utt, spk_id, chapter_no, utt_no in tqdm(dataset): - sample_id = f"{spk_id}-{chapter_no}-{utt_no}" - manifest["id"].append(sample_id) - manifest["audio"].append(audio_paths[sample_id]) - manifest["n_frames"].append(audio_lengths[sample_id]) - manifest["tgt_text"].append(utt.lower()) - manifest["speaker"].append(spk_id) - save_df_to_tsv( - pd.DataFrame.from_dict(manifest), out_root / f"{split}.tsv" - ) - if split.startswith("train"): - train_text.extend(manifest["tgt_text"]) - # Generate vocab - vocab_size = "" if args.vocab_type == "char" else str(args.vocab_size) - spm_filename_prefix = f"spm_{args.vocab_type}{vocab_size}" - with NamedTemporaryFile(mode="w") as f: - for t in train_text: - f.write(t + "\n") - gen_vocab( - Path(f.name), - out_root / spm_filename_prefix, - args.vocab_type, - args.vocab_size, - ) - # Generate config YAML - gen_config_yaml( - out_root, - spm_filename=spm_filename_prefix + ".model", - specaugment_policy="ld" - ) - # Clean up - shutil.rmtree(feature_root) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--output-root", "-o", required=True, type=str) - parser.add_argument( - "--vocab-type", - default="unigram", - required=True, - type=str, - choices=["bpe", "unigram", "char"], - ), - parser.add_argument("--vocab-size", default=10000, type=int) - args = parser.parse_args() - - process(args) - - -if __name__ == "__main__": - main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_text/prep_mtedx_data.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_text/prep_mtedx_data.py deleted file mode 100644 index 63e784bb8bcddcf84813d3e31fa11bab433a4fc5..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_text/prep_mtedx_data.py +++ /dev/null @@ -1,269 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import logging -import os -import shutil -from itertools import groupby -from pathlib import Path -from tempfile import NamedTemporaryFile -from typing import Tuple - -import pandas as pd -import soundfile as sf -import torch -from examples.speech_to_text.data_utils import ( - create_zip, - extract_fbank_features, - filter_manifest_df, - gen_config_yaml, - gen_vocab, - get_zip_manifest, - load_df_from_tsv, - save_df_to_tsv, -) -from fairseq.data.audio.audio_utils import convert_waveform, get_waveform -from torch.utils.data import Dataset -from tqdm import tqdm - -log = logging.getLogger(__name__) - - -MANIFEST_COLUMNS = [ - "id", "audio", "n_frames", "tgt_text", "speaker", "tgt_lang" -] - - -class mTEDx(Dataset): - """ - Create a Dataset for Multilingual TEDx. - Each item is a tuple of the form: waveform, sample_rate, source utterance, - target utterance, speaker_id, utterance_id - """ - - SPLITS = ["train", "valid", "test"] - LANGPAIRS = ["es-es", "fr-fr", "pt-pt", "it-it", "ru-ru", "el-el", "ar-ar", - "de-de", "es-en", "es-fr", "es-pt", "es-it", "fr-en", "fr-es", - "fr-pt", "pt-en", "pt-es", "it-en", "it-es", "ru-en", "el-en"] - - def __init__(self, root: str, lang: str, split: str) -> None: - assert split in self.SPLITS and lang in self.LANGPAIRS - _root = Path(root) / f"{lang}" / "data" / split - wav_root, txt_root = _root / "wav", _root / "txt" - assert _root.is_dir() and wav_root.is_dir() and txt_root.is_dir() - # Load audio segments - try: - import yaml - except ImportError: - print( - "Please install PyYAML to load the Multilingual TEDx YAML files" - ) - with open(txt_root / f"{split}.yaml") as f: - segments = yaml.load(f, Loader=yaml.BaseLoader) - # Load source and target utterances - src, tgt = lang.split("-") - for _lang in [src, tgt]: - with open(txt_root / f"{split}.{_lang}") as f: - utterances = [r.strip() for r in f] - assert len(segments) == len(utterances) - for i, u in enumerate(utterances): - segments[i][_lang] = u - # Gather info - self.data = [] - for wav_filename, _seg_group in groupby(segments, lambda x: x["wav"]): - wav_filename = wav_filename.replace(".wav", ".flac") - wav_path = wav_root / wav_filename - sample_rate = sf.info(wav_path.as_posix()).samplerate - seg_group = sorted(_seg_group, key=lambda x: float(x["offset"])) - for i, segment in enumerate(seg_group): - offset = int(float(segment["offset"]) * sample_rate) - n_frames = int(float(segment["duration"]) * sample_rate) - _id = f"{wav_path.stem}_{i}" - self.data.append( - ( - wav_path.as_posix(), - offset, - n_frames, - sample_rate, - segment[src], - segment[tgt], - segment["speaker_id"], - tgt, - _id, - ) - ) - - def __getitem__( - self, n: int - ) -> Tuple[torch.Tensor, int, str, str, str, str, str]: - wav_path, offset, n_frames, sr, src_utt, tgt_utt, spk_id, tgt_lang, \ - utt_id = self.data[n] - waveform, _ = get_waveform(wav_path, frames=n_frames, start=offset) - waveform = torch.from_numpy(waveform) - return waveform, sr, src_utt, tgt_utt, spk_id, tgt_lang, utt_id - - def __len__(self) -> int: - return len(self.data) - - -def process(args): - root = Path(args.data_root).absolute() - for lang in mTEDx.LANGPAIRS: - cur_root = root / f"{lang}" - if not cur_root.is_dir(): - print(f"{cur_root.as_posix()} does not exist. Skipped.") - continue - # Extract features - audio_root = cur_root / ("flac" if args.use_audio_input else "fbank80") - audio_root.mkdir(exist_ok=True) - for split in mTEDx.SPLITS: - print(f"Fetching split {split}...") - dataset = mTEDx(root.as_posix(), lang, split) - if args.use_audio_input: - print("Converting audios...") - for waveform, sample_rate, _, _, _, utt_id in tqdm(dataset): - tgt_sample_rate = 16_000 - _wavform, _ = convert_waveform( - waveform, sample_rate, to_mono=True, - to_sample_rate=tgt_sample_rate - ) - sf.write( - (audio_root / f"{utt_id}.flac").as_posix(), - _wavform.numpy(), tgt_sample_rate - ) - else: - print("Extracting log mel filter bank features...") - for waveform, sample_rate, _, _, _, _, utt_id in tqdm(dataset): - extract_fbank_features( - waveform, sample_rate, audio_root / f"{utt_id}.npy" - ) - # Pack features into ZIP - zip_path = cur_root / f"{audio_root.name}.zip" - print("ZIPing audios/features...") - create_zip(audio_root, zip_path) - print("Fetching ZIP manifest...") - audio_paths, audio_lengths = get_zip_manifest(zip_path) - # Generate TSV manifest - print("Generating manifest...") - train_text = [] - for split in mTEDx.SPLITS: - is_train_split = split.startswith("train") - manifest = {c: [] for c in MANIFEST_COLUMNS} - ds = mTEDx(args.data_root, lang, split) - for _, _, src_utt, tgt_utt, spk_id, tgt_lang, utt_id in tqdm(ds): - manifest["id"].append(utt_id) - manifest["audio"].append(audio_paths[utt_id]) - manifest["n_frames"].append(audio_lengths[utt_id]) - manifest["tgt_text"].append( - src_utt if args.task == "asr" else tgt_utt - ) - manifest["speaker"].append(spk_id) - manifest["tgt_lang"].append(tgt_lang) - if is_train_split: - train_text.extend(manifest["tgt_text"]) - df = pd.DataFrame.from_dict(manifest) - df = filter_manifest_df(df, is_train_split=is_train_split) - save_df_to_tsv(df, cur_root / f"{split}_{args.task}.tsv") - # Generate vocab - v_size_str = "" if args.vocab_type == "char" else str(args.vocab_size) - spm_filename_prefix = f"spm_{args.vocab_type}{v_size_str}_{args.task}" - with NamedTemporaryFile(mode="w") as f: - for t in train_text: - f.write(t + "\n") - gen_vocab( - Path(f.name), - cur_root / spm_filename_prefix, - args.vocab_type, - args.vocab_size, - ) - # Generate config YAML - if args.use_audio_input: - gen_config_yaml( - cur_root, - spm_filename=spm_filename_prefix + ".model", - yaml_filename=f"config_{args.task}.yaml", - specaugment_policy=None, - extra={"use_audio_input": True} - ) - else: - gen_config_yaml( - cur_root, - spm_filename=spm_filename_prefix + ".model", - yaml_filename=f"config_{args.task}.yaml", - specaugment_policy="lb", - ) - # Clean up - shutil.rmtree(audio_root) - - -def process_joint(args): - cur_root = Path(args.data_root) - assert all((cur_root / f"{lang}").is_dir() for lang in mTEDx.LANGPAIRS), \ - "do not have downloaded data available for all languages" - # Generate vocab - vocab_size_str = "" if args.vocab_type == "char" else str(args.vocab_size) - spm_filename_prefix = f"spm_{args.vocab_type}{vocab_size_str}_{args.task}" - with NamedTemporaryFile(mode="w") as f: - for lang in mTEDx.LANGPAIRS: - tsv_path = cur_root / f"{lang}" / f"train_{args.task}.tsv" - df = load_df_from_tsv(tsv_path) - for t in df["tgt_text"]: - f.write(t + "\n") - special_symbols = None - if args.joint: - # Add tgt_lang tags to dict - special_symbols = list( - {f'' for lang in mTEDx.LANGPAIRS} - ) - gen_vocab( - Path(f.name), - cur_root / spm_filename_prefix, - args.vocab_type, - args.vocab_size, - special_symbols=special_symbols - ) - # Generate config YAML - gen_config_yaml( - cur_root, - spm_filename=spm_filename_prefix + ".model", - yaml_filename=f"config_{args.task}.yaml", - specaugment_policy="ld", - prepend_tgt_lang_tag=(args.joint), - ) - # Make symbolic links to manifests - for lang in mTEDx.LANGPAIRS: - for split in mTEDx.SPLITS: - src_path = cur_root / f"{lang}" / f"{split}_{args.task}.tsv" - desc_path = cur_root / f"{split}_{lang}_{args.task}.tsv" - if not desc_path.is_symlink(): - os.symlink(src_path, desc_path) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--data-root", "-d", required=True, type=str) - parser.add_argument( - "--vocab-type", - default="unigram", - required=True, - type=str, - choices=["bpe", "unigram", "char"], - ), - parser.add_argument("--vocab-size", default=8000, type=int) - parser.add_argument("--task", type=str, choices=["asr", "st"]) - parser.add_argument("--joint", action="store_true", help="") - parser.add_argument("--use-audio-input", action="store_true") - args = parser.parse_args() - - if args.joint: - process_joint(args) - else: - process(args) - - -if __name__ == "__main__": - main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_text/prep_mustc_data.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_text/prep_mustc_data.py deleted file mode 100644 index e0d830f1b1c84eaf088d8a288c22a5e85187f631..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_text/prep_mustc_data.py +++ /dev/null @@ -1,292 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import logging -import os -import shutil -from itertools import groupby -from pathlib import Path -from tempfile import NamedTemporaryFile -from typing import Tuple - -import numpy as np -import pandas as pd -import soundfile as sf -import torch -from examples.speech_to_text.data_utils import ( - cal_gcmvn_stats, - create_zip, - extract_fbank_features, - filter_manifest_df, - gen_config_yaml, - gen_vocab, - get_zip_manifest, - load_df_from_tsv, - save_df_to_tsv, -) -from fairseq.data.audio.audio_utils import convert_waveform, get_waveform -from torch.utils.data import Dataset -from tqdm import tqdm - -log = logging.getLogger(__name__) - - -MANIFEST_COLUMNS = ["id", "audio", "n_frames", "tgt_text", "speaker"] - - -class MUSTC(Dataset): - """ - Create a Dataset for MuST-C. Each item is a tuple of the form: - waveform, sample_rate, source utterance, target utterance, speaker_id, - utterance_id - """ - - SPLITS = ["train", "dev", "tst-COMMON", "tst-HE"] - LANGUAGES = ["de", "es", "fr", "it", "nl", "pt", "ro", "ru"] - - def __init__(self, root: str, lang: str, split: str) -> None: - assert split in self.SPLITS and lang in self.LANGUAGES - _root = Path(root) / f"en-{lang}" / "data" / split - wav_root, txt_root = _root / "wav", _root / "txt" - assert _root.is_dir() and wav_root.is_dir() and txt_root.is_dir() - # Load audio segments - try: - import yaml - except ImportError: - print("Please install PyYAML to load the MuST-C YAML files") - with open(txt_root / f"{split}.yaml") as f: - segments = yaml.load(f, Loader=yaml.BaseLoader) - # Load source and target utterances - for _lang in ["en", lang]: - with open(txt_root / f"{split}.{_lang}") as f: - utterances = [r.strip() for r in f] - assert len(segments) == len(utterances) - for i, u in enumerate(utterances): - segments[i][_lang] = u - # Gather info - self.data = [] - for wav_filename, _seg_group in groupby(segments, lambda x: x["wav"]): - wav_path = wav_root / wav_filename - sample_rate = sf.info(wav_path.as_posix()).samplerate - seg_group = sorted(_seg_group, key=lambda x: x["offset"]) - for i, segment in enumerate(seg_group): - offset = int(float(segment["offset"]) * sample_rate) - n_frames = int(float(segment["duration"]) * sample_rate) - _id = f"{wav_path.stem}_{i}" - self.data.append( - ( - wav_path.as_posix(), - offset, - n_frames, - sample_rate, - segment["en"], - segment[lang], - segment["speaker_id"], - _id, - ) - ) - - def __getitem__( - self, n: int - ) -> Tuple[torch.Tensor, int, str, str, str, str]: - wav_path, offset, n_frames, sr, src_utt, tgt_utt, spk_id, \ - utt_id = self.data[n] - waveform, _ = get_waveform(wav_path, frames=n_frames, start=offset) - waveform = torch.from_numpy(waveform) - return waveform, sr, src_utt, tgt_utt, spk_id, utt_id - - def __len__(self) -> int: - return len(self.data) - - -def process(args): - root = Path(args.data_root).absolute() - for lang in MUSTC.LANGUAGES: - cur_root = root / f"en-{lang}" - if not cur_root.is_dir(): - print(f"{cur_root.as_posix()} does not exist. Skipped.") - continue - # Extract features - audio_root = cur_root / ("flac" if args.use_audio_input else "fbank80") - audio_root.mkdir(exist_ok=True) - - for split in MUSTC.SPLITS: - print(f"Fetching split {split}...") - dataset = MUSTC(root.as_posix(), lang, split) - if args.use_audio_input: - print("Converting audios...") - for waveform, sample_rate, _, _, _, utt_id in tqdm(dataset): - tgt_sample_rate = 16_000 - _wavform, _ = convert_waveform( - waveform, sample_rate, to_mono=True, - to_sample_rate=tgt_sample_rate - ) - sf.write( - (audio_root / f"{utt_id}.flac").as_posix(), - _wavform.T.numpy(), tgt_sample_rate - ) - else: - print("Extracting log mel filter bank features...") - gcmvn_feature_list = [] - if split == 'train' and args.cmvn_type == "global": - print("And estimating cepstral mean and variance stats...") - - for waveform, sample_rate, _, _, _, utt_id in tqdm(dataset): - features = extract_fbank_features( - waveform, sample_rate, audio_root / f"{utt_id}.npy" - ) - if split == 'train' and args.cmvn_type == "global": - if len(gcmvn_feature_list) < args.gcmvn_max_num: - gcmvn_feature_list.append(features) - - if split == 'train' and args.cmvn_type == "global": - # Estimate and save cmv - stats = cal_gcmvn_stats(gcmvn_feature_list) - with open(cur_root / "gcmvn.npz", "wb") as f: - np.savez(f, mean=stats["mean"], std=stats["std"]) - - # Pack features into ZIP - zip_path = cur_root / f"{audio_root.name}.zip" - print("ZIPing audios/features...") - create_zip(audio_root, zip_path) - print("Fetching ZIP manifest...") - audio_paths, audio_lengths = get_zip_manifest( - zip_path, - is_audio=args.use_audio_input, - ) - # Generate TSV manifest - print("Generating manifest...") - train_text = [] - for split in MUSTC.SPLITS: - is_train_split = split.startswith("train") - manifest = {c: [] for c in MANIFEST_COLUMNS} - dataset = MUSTC(args.data_root, lang, split) - for _, _, src_utt, tgt_utt, speaker_id, utt_id in tqdm(dataset): - manifest["id"].append(utt_id) - manifest["audio"].append(audio_paths[utt_id]) - manifest["n_frames"].append(audio_lengths[utt_id]) - manifest["tgt_text"].append( - src_utt if args.task == "asr" else tgt_utt - ) - manifest["speaker"].append(speaker_id) - if is_train_split: - train_text.extend(manifest["tgt_text"]) - df = pd.DataFrame.from_dict(manifest) - df = filter_manifest_df(df, is_train_split=is_train_split) - save_df_to_tsv(df, cur_root / f"{split}_{args.task}.tsv") - # Generate vocab - v_size_str = "" if args.vocab_type == "char" else str(args.vocab_size) - spm_filename_prefix = f"spm_{args.vocab_type}{v_size_str}_{args.task}" - with NamedTemporaryFile(mode="w") as f: - for t in train_text: - f.write(t + "\n") - gen_vocab( - Path(f.name), - cur_root / spm_filename_prefix, - args.vocab_type, - args.vocab_size, - ) - # Generate config YAML - if args.use_audio_input: - gen_config_yaml( - cur_root, - spm_filename=spm_filename_prefix + ".model", - yaml_filename=f"config_{args.task}.yaml", - specaugment_policy=None, - extra={"use_audio_input": True} - ) - else: - gen_config_yaml( - cur_root, - spm_filename=spm_filename_prefix + ".model", - yaml_filename=f"config_{args.task}.yaml", - specaugment_policy="lb", - cmvn_type=args.cmvn_type, - gcmvn_path=( - cur_root / "gcmvn.npz" if args.cmvn_type == "global" - else None - ), - ) - # Clean up - shutil.rmtree(audio_root) - - -def process_joint(args): - cur_root = Path(args.data_root) - assert all( - (cur_root / f"en-{lang}").is_dir() for lang in MUSTC.LANGUAGES - ), "do not have downloaded data available for all 8 languages" - # Generate vocab - vocab_size_str = "" if args.vocab_type == "char" else str(args.vocab_size) - spm_filename_prefix = f"spm_{args.vocab_type}{vocab_size_str}_{args.task}" - with NamedTemporaryFile(mode="w") as f: - for lang in MUSTC.LANGUAGES: - tsv_path = cur_root / f"en-{lang}" / f"train_{args.task}.tsv" - df = load_df_from_tsv(tsv_path) - for t in df["tgt_text"]: - f.write(t + "\n") - special_symbols = None - if args.task == 'st': - special_symbols = [f'' for lang in MUSTC.LANGUAGES] - gen_vocab( - Path(f.name), - cur_root / spm_filename_prefix, - args.vocab_type, - args.vocab_size, - special_symbols=special_symbols - ) - # Generate config YAML - gen_config_yaml( - cur_root, - spm_filename=spm_filename_prefix + ".model", - yaml_filename=f"config_{args.task}.yaml", - specaugment_policy="ld", - prepend_tgt_lang_tag=(args.task == "st"), - ) - # Make symbolic links to manifests - for lang in MUSTC.LANGUAGES: - for split in MUSTC.SPLITS: - src_path = cur_root / f"en-{lang}" / f"{split}_{args.task}.tsv" - desc_path = cur_root / f"{split}_{lang}_{args.task}.tsv" - if not desc_path.is_symlink(): - os.symlink(src_path, desc_path) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--data-root", "-d", required=True, type=str) - parser.add_argument( - "--vocab-type", - default="unigram", - required=True, - type=str, - choices=["bpe", "unigram", "char"], - ), - parser.add_argument("--vocab-size", default=8000, type=int) - parser.add_argument("--task", type=str, choices=["asr", "st"]) - parser.add_argument("--joint", action="store_true", help="") - parser.add_argument( - "--cmvn-type", default="utterance", - choices=["global", "utterance"], - help="The type of cepstral mean and variance normalization" - ) - parser.add_argument( - "--gcmvn-max-num", default=150000, type=int, - help="Maximum number of sentences to use to estimate global mean and " - "variance" - ) - parser.add_argument("--use-audio-input", action="store_true") - args = parser.parse_args() - - if args.joint: - process_joint(args) - else: - process(args) - - -if __name__ == "__main__": - main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_text/seg_mustc_data.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_text/seg_mustc_data.py deleted file mode 100644 index 7a9cb90a7cf3e3639ab2dc9ea102a2ed278b7e46..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_text/seg_mustc_data.py +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import logging -from pathlib import Path - -import soundfile as sf -from examples.speech_to_text.prep_mustc_data import MUSTC -from tqdm import tqdm - -log = logging.getLogger(__name__) - - -def main(args): - root = Path(args.data_root).absolute() - lang = args.lang - split = args.split - - cur_root = root / f"en-{lang}" - assert cur_root.is_dir(), ( - f"{cur_root.as_posix()} does not exist. Skipped." - ) - - dataset = MUSTC(root.as_posix(), lang, split) - output = Path(args.output).absolute() - output.mkdir(exist_ok=True) - f_text = open(output / f"{split}.{lang}", "w") - f_wav_list = open(output / f"{split}.wav_list", "w") - for waveform, sample_rate, _, text, _, utt_id in tqdm(dataset): - sf.write( - output / f"{utt_id}.wav", - waveform.squeeze(0).numpy(), - samplerate=int(sample_rate) - ) - f_text.write(text + "\n") - f_wav_list.write(str(output / f"{utt_id}.wav") + "\n") - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("--data-root", "-d", required=True, type=str) - parser.add_argument("--task", required=True, type=str, choices=["asr", "st"]) - parser.add_argument("--lang", required=True, type=str) - parser.add_argument("--output", required=True, type=str) - parser.add_argument("--split", required=True, choices=MUSTC.SPLITS) - args = parser.parse_args() - - main(args) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_text/simultaneous_translation/agents/fairseq_simul_st_agent.py b/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_text/simultaneous_translation/agents/fairseq_simul_st_agent.py deleted file mode 100644 index ee8046c4d635a5a4034e9a43d0c72ea2895abf1f..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/speech_to_text/simultaneous_translation/agents/fairseq_simul_st_agent.py +++ /dev/null @@ -1,364 +0,0 @@ -import json -import math -import os - -import numpy as np -import torch -import torchaudio.compliance.kaldi as kaldi -import yaml -from fairseq import checkpoint_utils, tasks -from fairseq.file_io import PathManager - -try: - from simuleval import DEFAULT_EOS, READ_ACTION, WRITE_ACTION - from simuleval.agents import SpeechAgent - from simuleval.states import ListEntry, SpeechStates -except ImportError: - print("Please install simuleval 'pip install simuleval'") - -SHIFT_SIZE = 10 -WINDOW_SIZE = 25 -SAMPLE_RATE = 16000 -FEATURE_DIM = 80 -BOW_PREFIX = "\u2581" - - -class OnlineFeatureExtractor: - """ - Extract speech feature on the fly. - """ - - def __init__(self, args): - self.shift_size = args.shift_size - self.window_size = args.window_size - assert self.window_size >= self.shift_size - - self.sample_rate = args.sample_rate - self.feature_dim = args.feature_dim - self.num_samples_per_shift = int(self.shift_size * self.sample_rate / 1000) - self.num_samples_per_window = int(self.window_size * self.sample_rate / 1000) - self.len_ms_to_samples = lambda x: x * self.sample_rate / 1000 - self.previous_residual_samples = [] - self.global_cmvn = args.global_cmvn - - def clear_cache(self): - self.previous_residual_samples = [] - - def __call__(self, new_samples): - samples = self.previous_residual_samples + new_samples - if len(samples) < self.num_samples_per_window: - self.previous_residual_samples = samples - return - - # num_frames is the number of frames from the new segment - num_frames = math.floor( - (len(samples) - self.len_ms_to_samples(self.window_size - self.shift_size)) - / self.num_samples_per_shift - ) - - # the number of frames used for feature extraction - # including some part of thte previous segment - effective_num_samples = int( - num_frames * self.len_ms_to_samples(self.shift_size) - + self.len_ms_to_samples(self.window_size - self.shift_size) - ) - - input_samples = samples[:effective_num_samples] - self.previous_residual_samples = samples[ - num_frames * self.num_samples_per_shift: - ] - - torch.manual_seed(1) - output = kaldi.fbank( - torch.FloatTensor(input_samples).unsqueeze(0), - num_mel_bins=self.feature_dim, - frame_length=self.window_size, - frame_shift=self.shift_size, - ).numpy() - - output = self.transform(output) - - return torch.from_numpy(output) - - def transform(self, input): - if self.global_cmvn is None: - return input - - mean = self.global_cmvn["mean"] - std = self.global_cmvn["std"] - - x = np.subtract(input, mean) - x = np.divide(x, std) - return x - - -class TensorListEntry(ListEntry): - """ - Data structure to store a list of tensor. - """ - - def append(self, value): - - if len(self.value) == 0: - self.value = value - return - - self.value = torch.cat([self.value] + [value], dim=0) - - def info(self): - return { - "type": str(self.new_value_type), - "length": self.__len__(), - "value": "" if type(self.value) is list else self.value.size(), - } - - -class FairseqSimulSTAgent(SpeechAgent): - - speech_segment_size = 40 # in ms, 4 pooling ratio * 10 ms step size - - def __init__(self, args): - super().__init__(args) - - self.eos = DEFAULT_EOS - - self.gpu = getattr(args, "gpu", False) - - self.args = args - - self.load_model_vocab(args) - - if getattr( - self.model.decoder.layers[0].encoder_attn, - 'pre_decision_ratio', - None - ) is not None: - self.speech_segment_size *= ( - self.model.decoder.layers[0].encoder_attn.pre_decision_ratio - ) - - args.global_cmvn = None - if args.config: - with open(os.path.join(args.data_bin, args.config), "r") as f: - config = yaml.load(f, Loader=yaml.BaseLoader) - - if "global_cmvn" in config: - args.global_cmvn = np.load(config["global_cmvn"]["stats_npz_path"]) - - if args.global_stats: - with PathManager.open(args.global_stats, "r") as f: - global_cmvn = json.loads(f.read()) - self.global_cmvn = {"mean": global_cmvn["mean"], "std": global_cmvn["stddev"]} - - self.feature_extractor = OnlineFeatureExtractor(args) - - self.max_len = args.max_len - - self.force_finish = args.force_finish - - torch.set_grad_enabled(False) - - def build_states(self, args, client, sentence_id): - # Initialize states here, for example add customized entry to states - # This function will be called at beginning of every new sentence - states = SpeechStates(args, client, sentence_id, self) - self.initialize_states(states) - return states - - def to_device(self, tensor): - if self.gpu: - return tensor.cuda() - else: - return tensor.cpu() - - @staticmethod - def add_args(parser): - # fmt: off - parser.add_argument('--model-path', type=str, required=True, - help='path to your pretrained model.') - parser.add_argument("--data-bin", type=str, required=True, - help="Path of data binary") - parser.add_argument("--config", type=str, default=None, - help="Path to config yaml file") - parser.add_argument("--global-stats", type=str, default=None, - help="Path to json file containing cmvn stats") - parser.add_argument("--tgt-splitter-type", type=str, default="SentencePiece", - help="Subword splitter type for target text") - parser.add_argument("--tgt-splitter-path", type=str, default=None, - help="Subword splitter model path for target text") - parser.add_argument("--user-dir", type=str, default="examples/simultaneous_translation", - help="User directory for simultaneous translation") - parser.add_argument("--max-len", type=int, default=200, - help="Max length of translation") - parser.add_argument("--force-finish", default=False, action="store_true", - help="Force the model to finish the hypothsis if the source is not finished") - parser.add_argument("--shift-size", type=int, default=SHIFT_SIZE, - help="Shift size of feature extraction window.") - parser.add_argument("--window-size", type=int, default=WINDOW_SIZE, - help="Window size of feature extraction window.") - parser.add_argument("--sample-rate", type=int, default=SAMPLE_RATE, - help="Sample rate") - parser.add_argument("--feature-dim", type=int, default=FEATURE_DIM, - help="Acoustic feature dimension.") - - # fmt: on - return parser - - def load_model_vocab(self, args): - - filename = args.model_path - if not os.path.exists(filename): - raise IOError("Model file not found: {}".format(filename)) - - state = checkpoint_utils.load_checkpoint_to_cpu(filename) - - task_args = state["cfg"]["task"] - task_args.data = args.data_bin - - if args.config is not None: - task_args.config_yaml = args.config - - task = tasks.setup_task(task_args) - - # build model for ensemble - state["cfg"]["model"].load_pretrained_encoder_from = None - state["cfg"]["model"].load_pretrained_decoder_from = None - self.model = task.build_model(state["cfg"]["model"]) - self.model.load_state_dict(state["model"], strict=True) - self.model.eval() - self.model.share_memory() - - if self.gpu: - self.model.cuda() - - # Set dictionary - self.dict = {} - self.dict["tgt"] = task.target_dictionary - - def initialize_states(self, states): - self.feature_extractor.clear_cache() - states.units.source = TensorListEntry() - states.units.target = ListEntry() - states.incremental_states = dict() - - def segment_to_units(self, segment, states): - # Convert speech samples to features - features = self.feature_extractor(segment) - if features is not None: - return [features] - else: - return [] - - def units_to_segment(self, units, states): - # Merge sub word to full word. - if self.model.decoder.dictionary.eos() == units[0]: - return DEFAULT_EOS - - segment = [] - if None in units.value: - units.value.remove(None) - - for index in units: - if index is None: - units.pop() - token = self.model.decoder.dictionary.string([index]) - if token.startswith(BOW_PREFIX): - if len(segment) == 0: - segment += [token.replace(BOW_PREFIX, "")] - else: - for j in range(len(segment)): - units.pop() - - string_to_return = ["".join(segment)] - - if self.model.decoder.dictionary.eos() == units[0]: - string_to_return += [DEFAULT_EOS] - - return string_to_return - else: - segment += [token.replace(BOW_PREFIX, "")] - - if ( - len(units) > 0 - and self.model.decoder.dictionary.eos() == units[-1] - or len(states.units.target) > self.max_len - ): - tokens = [self.model.decoder.dictionary.string([unit]) for unit in units] - return ["".join(tokens).replace(BOW_PREFIX, "")] + [DEFAULT_EOS] - - return None - - def update_model_encoder(self, states): - if len(states.units.source) == 0: - return - src_indices = self.to_device( - states.units.source.value.unsqueeze(0) - ) - src_lengths = self.to_device( - torch.LongTensor([states.units.source.value.size(0)]) - ) - - states.encoder_states = self.model.encoder(src_indices, src_lengths) - torch.cuda.empty_cache() - - def update_states_read(self, states): - # Happens after a read action. - self.update_model_encoder(states) - - def policy(self, states): - if not getattr(states, "encoder_states", None): - return READ_ACTION - - tgt_indices = self.to_device( - torch.LongTensor( - [self.model.decoder.dictionary.eos()] - + [x for x in states.units.target.value if x is not None] - ).unsqueeze(0) - ) - - states.incremental_states["steps"] = { - "src": states.encoder_states["encoder_out"][0].size(0), - "tgt": 1 + len(states.units.target), - } - - states.incremental_states["online"] = {"only": torch.tensor(not states.finish_read())} - - x, outputs = self.model.decoder.forward( - prev_output_tokens=tgt_indices, - encoder_out=states.encoder_states, - incremental_state=states.incremental_states, - ) - - states.decoder_out = x - - states.decoder_out_extra = outputs - - torch.cuda.empty_cache() - - if outputs.action == 0: - return READ_ACTION - else: - return WRITE_ACTION - - def predict(self, states): - decoder_states = states.decoder_out - - lprobs = self.model.get_normalized_probs( - [decoder_states[:, -1:]], log_probs=True - ) - - index = lprobs.argmax(dim=-1) - - index = index[0, 0].item() - - if ( - self.force_finish - and index == self.model.decoder.dictionary.eos() - and not states.finish_read() - ): - # If we want to force finish the translation - # (don't stop before finish reading), return a None - # self.model.decoder.clear_cache(states.incremental_states) - index = None - - return index diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/stories/README.md b/audio/safeear/safeear_repo/fairseq_ours/examples/stories/README.md deleted file mode 100644 index 588941eddc5f0280f5254affd40ef49de874c885..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/stories/README.md +++ /dev/null @@ -1,66 +0,0 @@ -# Hierarchical Neural Story Generation (Fan et al., 2018) - -The following commands provide an example of pre-processing data, training a model, and generating text for story generation with the WritingPrompts dataset. - -## Pre-trained models - -Description | Dataset | Model | Test set(s) ----|---|---|--- -Stories with Convolutional Model
([Fan et al., 2018](https://arxiv.org/abs/1805.04833)) | [WritingPrompts](https://dl.fbaipublicfiles.com/fairseq/data/writingPrompts.tar.gz) | [download (.tar.bz2)](https://dl.fbaipublicfiles.com/fairseq/models/stories_checkpoint.tar.bz2) | [download (.tar.bz2)](https://dl.fbaipublicfiles.com/fairseq/data/stories_test.tar.bz2) - -We provide sample stories generated by the [convolutional seq2seq model](https://dl.fbaipublicfiles.com/fairseq/data/seq2seq_stories.txt) and [fusion model](https://dl.fbaipublicfiles.com/fairseq/data/fusion_stories.txt) from [Fan et al., 2018](https://arxiv.org/abs/1805.04833). The corresponding prompts for the fusion model can be found [here](https://dl.fbaipublicfiles.com/fairseq/data/fusion_prompts.txt). Note that there are unk in the file, as we modeled a small full vocabulary (no BPE or pre-training). We did not use these unk prompts for human evaluation. - -## Dataset - -The dataset can be downloaded like this: - -```bash -cd examples/stories -curl https://dl.fbaipublicfiles.com/fairseq/data/writingPrompts.tar.gz | tar xvzf - -``` - -and contains a train, test, and valid split. The dataset is described here: https://arxiv.org/abs/1805.04833. We model only the first 1000 words of each story, including one newLine token. - -## Example usage - -First we will preprocess the dataset. Note that the dataset release is the full data, but the paper models the first 1000 words of each story. Here is example code that trims the dataset to the first 1000 words of each story: -```python -data = ["train", "test", "valid"] -for name in data: - with open(name + ".wp_target") as f: - stories = f.readlines() - stories = [" ".join(i.split()[0:1000]) for i in stories] - with open(name + ".wp_target", "w") as o: - for line in stories: - o.write(line.strip() + "\n") -``` - -Once we've trimmed the data we can binarize it and train our model: -```bash -# Binarize the dataset: -export TEXT=examples/stories/writingPrompts -fairseq-preprocess --source-lang wp_source --target-lang wp_target \ - --trainpref $TEXT/train --validpref $TEXT/valid --testpref $TEXT/test \ - --destdir data-bin/writingPrompts --padding-factor 1 --thresholdtgt 10 --thresholdsrc 10 - -# Train the model: -fairseq-train data-bin/writingPrompts -a fconv_self_att_wp --lr 0.25 --optimizer nag --clip-norm 0.1 --max-tokens 1500 --lr-scheduler reduce_lr_on_plateau --decoder-attention True --encoder-attention False --criterion label_smoothed_cross_entropy --weight-decay .0000001 --label-smoothing 0 --source-lang wp_source --target-lang wp_target --gated-attention True --self-attention True --project-input True --pretrained False - -# Train a fusion model: -# add the arguments: --pretrained True --pretrained-checkpoint path/to/checkpoint - -# Generate: -# Note: to load the pretrained model at generation time, you need to pass in a model-override argument to communicate to the fusion model at generation time where you have placed the pretrained checkpoint. By default, it will load the exact path of the fusion model's pretrained model from training time. You should use model-override if you have moved the pretrained model (or are using our provided models). If you are generating from a non-fusion model, the model-override argument is not necessary. - -fairseq-generate data-bin/writingPrompts --path /path/to/trained/model/checkpoint_best.pt --batch-size 32 --beam 1 --sampling --sampling-topk 10 --temperature 0.8 --nbest 1 --model-overrides "{'pretrained_checkpoint':'/path/to/pretrained/model/checkpoint'}" -``` - -## Citation -```bibtex -@inproceedings{fan2018hierarchical, - title = {Hierarchical Neural Story Generation}, - author = {Fan, Angela and Lewis, Mike and Dauphin, Yann}, - booktitle = {Conference of the Association for Computational Linguistics (ACL)}, - year = 2018, -} -``` diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/README.md b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/README.md deleted file mode 100644 index 7a76ffd57c066c20af94aa3fca24c18e2ba4c3dd..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/README.md +++ /dev/null @@ -1,21 +0,0 @@ -# Generative Spoken Language Modeling - -* [Paper](https://arxiv.org/abs/2102.01192) -* [Demo](https://speechbot.github.io/gslm/index.html) - -We build and evaluate generative speech2speech systems using [Log Mel Filtebank](https://pytorch.org/audio/stable/compliance.kaldi.html#fbank), [Modified CPC](https://github.com/facebookresearch/CPC_audio), [HuBERT Base](https://github.com/pytorch/fairseq/tree/main/examples/hubert) and [Wav2Vec 2.0 Large](https://github.com/pytorch/fairseq/tree/main/examples/wav2vec). Our system is composed of three components, namely, *speech2unit*, *ulm* and *unit2speech*. We explain about models and usage of these components in their respective sub-directories. See the links below. - -## Speech to Unit Model (speech2unit) -Speech to unit model is used for quantizing raw speech into learned discrete speech units. [More details](speech2unit) - -## Unit Language Model (ulm) -Unit Language Model is a generative language model trained on discrete speech units. [More details](ulm) - -## Unit to Speech Model (unit2speech) -Unit to speech model is used for synthesizing speech from discrete speech units. [More details](unit2speech) - -## Metrics -We show how to compute ASR based metrics as well as zero-shot metrics proposed in our paper [here](metrics). - -## Tools -We share two tools to resynthesize a given spoken utterance, and generate novel spoken language given a spoken prompt. [More detail](tools) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/metrics/README.md b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/metrics/README.md deleted file mode 100644 index 0a63e2f0d844ce157f9502c82738aac2a0de3f0c..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/metrics/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# GSLM Metrics - -## ASR Metrics -The suite of metrics here uses an ASR model to transcribe the synthesized speech into text, and then uses text-based metrics. We also use word error rate from ASR transcription itself as one of the metrics. [More details](asr_metrics) - -## ABX Metrics -We use [ABX](https://www.semanticscholar.org/paper/ABX-Discriminability-Measures-and-Applications-Schatz/13d3537228f728c1063cc83743cb118bba3367a0) to evaluate how well-separated phonetic categories are with quantized representations. [More details](abx_metrics) - -## sWUGGY and sBLIMP -We refer to [ZeroSpeech challenge](https://www.zerospeech.com/2021/track_s.html#scoring-based-metrics) for details on the sWUGGY and sBLIMP metrics. diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/metrics/abx_metrics/README.md b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/metrics/abx_metrics/README.md deleted file mode 100644 index aa2560f0453403fb5846c387848c78b037c79cb2..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/metrics/abx_metrics/README.md +++ /dev/null @@ -1,77 +0,0 @@ -# ABX-based evaluation - -ABX is used to evaluate the quality of the obtained discrete units. - -The life cycle of the ABX-based evaluation for the Speech-to-Unit contains the following steps: -1. Training an acoustic model (or use an existing acoustic model) ([description](./../..)) -2. Perform quantization of speech by learning a K-means clustering model ([description](./../..)) -3. Compute discrete features for ABX computation using the learned clusters -4. Compute the ABX score over the discrete features taking advantage of [libri-light's ABX evaluation script][ll-abx] - -Here we assume that you already went throught the first two steps and focus solely on extracting features and computing ABX scores. - -## Libri-light setup - -Follow [libri-light's instructions][ll-instructions] for installation and [ABX evaluation setup][ll-abx] (including the download of the data items required for ABX computation). - -## Computing ABX - -### Dumping quantized features - -The first step for the ABX computation is to dump the quantized representations corresponding to the test files. - -```shell -TYPE="hubert" -LAYER=6 -CKPT_PATH="" -KM_MODEL_PATH="" - -SUBSET="dev-clean" -MANIFEST="" -DATA_DIR="/$SUBSET" - -PYTHONPATH=. python examples/textless_nlp/gslm/metrics/abx_metrics/dump_abx_feats.py \ - --feature_type $TYPE \ - --kmeans_model_path $KM_MODEL_PATH \ - --checkpoint_path $CKPT_PATH \ - --layer $LAYER \ - --manifest_path $MANIFEST \ - --out_dir_path $DATA_DIR \ - --extension ".flac" -``` - -Again the manifest file follows the same structure than elsewhere in the codebase. - -### Compute ABX with Libri-light - -Use libri-light's `eval_ABX.py` script (within the appropriate environment set up) as followed: - -```shell -LIBRILIGHT_ROOT="" - -SUBSET="dev-clean" -DATA_DIR="/$SUBSET" -ITEM_FILE_PATH="$LIBRILIGHT_ROOT/eval/ABX_data/$SUBSET.item" -OUT_DIR="/$SUBSET" - -FILE_EXTENSION=".npy" -FEATURE_SIZE=0.02 # depends on the model used - -PYTHONPATH=$LIBRILIGHT_ROOT \ - python $LIBRILIGHT_ROOT/eval/eval_ABX.py \ - $DATA_DIR \ - $ITEM_FILE_PATH \ - --file_extension $FILE_EXTENSION \ - --feature_size $FEATURE_SIZE \ - --out $OUT_DIR \ - --mode "all" -``` - -Note that `FEATURE_SIZE` will depend on the model type you are using to extract the acoustic features: -* For HuBERT and Wav2Vec2.0, use `FEATURE_SIZE=0.02` -* For CPC and Log Mel, use `FEATURE_SIZE=0.01` - -If you have a gpu available, make sure you add the `--cuda` flag for faster computation. - -[ll-instructions]: https://github.com/facebookresearch/libri-light -[ll-abx]: https://github.com/facebookresearch/libri-light/tree/master/eval#abx diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/metrics/abx_metrics/dump_abx_feats.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/metrics/abx_metrics/dump_abx_feats.py deleted file mode 100644 index 214cc38c49827957373ec7d6f461dccf01fc0bc8..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/metrics/abx_metrics/dump_abx_feats.py +++ /dev/null @@ -1,107 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import logging -import os - -import joblib -import numpy as np -from examples.textless_nlp.gslm.speech2unit.clustering.utils import get_audio_files -from examples.textless_nlp.gslm.speech2unit.pretrained.utils import get_features - - -def get_logger(): - log_format = "[%(asctime)s] [%(levelname)s]: %(message)s" - logging.basicConfig(format=log_format, level=logging.INFO) - logger = logging.getLogger(__name__) - return logger - -def get_parser(): - parser = argparse.ArgumentParser( - description="Quantize using K-means clustering over acoustic features." - ) - parser.add_argument( - "--feature_type", - type=str, - choices=["logmel", "hubert", "w2v2", "cpc"], - default=None, - required=True, - help="Acoustic feature type", - ) - parser.add_argument( - "--kmeans_model_path", - type=str, - required=True, - help="K-means model file path to use for inference", - ) - parser.add_argument( - "--manifest_path", - type=str, - default=None, - help="Manifest file containing the root dir and file names", - ) - parser.add_argument( - "--checkpoint_path", - type=str, - help="Pretrained model checkpoint", - ) - parser.add_argument( - "--layer", - type=int, - help="The layer of the pretrained model to extract features from", - default=-1, - ) - parser.add_argument( - "--out_dir_path", - required=True, - type=str, - help="File path of quantized output.", - ) - parser.add_argument( - "--extension", type=str, default=".flac", help="Features file path" - ) - return parser - - -def one_hot(feat, n_clusters): - return np.eye(n_clusters)[feat] - -def main(args, logger): - # Feature extraction - logger.info(f"Extracting {args.feature_type} acoustic features...") - features_batch = get_features( - feature_type=args.feature_type, - checkpoint_path=args.checkpoint_path, - layer=args.layer, - manifest_path=args.manifest_path, - sample_pct=1.0, - flatten=False, - ) - logger.info(f"Features extracted for {len(features_batch)} utterances.\n") - logger.info(f"Dimensionality of representation = {features_batch[0].shape[1]}") - - logger.info(f"Loading K-means model from {args.kmeans_model_path} ...") - kmeans_model = joblib.load(open(args.kmeans_model_path, "rb")) - kmeans_model.verbose = False - - _, fnames, _ = get_audio_files(args.manifest_path) - - os.makedirs(args.out_dir_path, exist_ok=True) - logger.info(f"Writing quantized features to {args.out_dir_path}") - for i, feats in enumerate(features_batch): - pred = kmeans_model.predict(feats) - emb = one_hot(pred, kmeans_model.n_clusters) - base_fname = os.path.basename(fnames[i]).rstrip(args.extension) - output_path = os.path.join(args.out_dir_path, f"{base_fname}.npy") - with open(output_path, "wb") as f: - np.save(f, emb) - -if __name__ == "__main__": - parser = get_parser() - args = parser.parse_args() - logger = get_logger() - logger.info(args) - main(args, logger) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/metrics/asr_metrics/README.md b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/metrics/asr_metrics/README.md deleted file mode 100644 index 90741f42b0b070f2a91b63c8badb817c6aa24230..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/metrics/asr_metrics/README.md +++ /dev/null @@ -1,87 +0,0 @@ -# ASR-based evaluation - -Overall, the life cycle of the ASR-based evaluation for an ULM contains the following steps: - 1. Training an ULM and sampling from it [[description]](./../../ulm) - 2. Running UTS on the sampled unit sequences [[description]](./../../unit2speech) - 3. Pre-processing for the ASR (down-sampling to 16 KHz, aligning length of the generated audio with ground-truth utterances) - 4. Running ASR - 5. Calculation of the post-ASR evaluation metrics - -Here we assume that you have already went throught the first two steps and focus on the rest. - -## Preprocessing -### Down-sampling to 16KHz -The bulk conversion can be done by running -```bash - python $FAIRSEQ_ROOT/examples/textless_nlp/gslm/unit2speech/convert_to_16k.py $UTS_OUTPUT $UTS_OUTPUT_DOWNSAMPLE - ``` - where `$UTS_OUTPUT` specifies the directory with the generated audio and `$UTS_OUTPUT_DOWNSAMPLE` is the directory where downsampled audio would be saved. - - ### Matching by length -This step is somewhat optional. However, if you want to compare the fluency and diversity of a generated speech utterance to that of the ground-truth speech with the same prefix, it is a good idea to force them to be of the same length. -```bash -python $FAIRSEQ_ROOT/examples/textless_nlp/asr_metrics/cut_as.py \ - --samples_dir=$UTS_OUTPUT_DOWNSAMPLE --out_dir=$UTS_OUTPUT_DOWNSAMPLE_CUT \ - --prompts_description=data/ground_truth_continuation_dev.json -``` - -Here `ground_truth_continuation_dev.json` is a json file with ground-truth text from LibriSpeech dev-clean, associated with some meta-data (assuming the evaluation is done on dev-clean). This file can be downloaded [[here]](https://dl.fbaipublicfiles.com/textless_nlp/gslm/eval_data/ground_truth_continuation_dev.json). A similar file for the test-clean is [[here]](https://dl.fbaipublicfiles.com/textless_nlp/gslm/eval_data/ground_truth_continuation_test.json). These files are used for the evaluation and contain texts for audio sequences that are at least 6s long. - -## Running ASR -We use a pre-trained wav2vec model to run the ASR step. We firstly need to prepare manifest files which, roughly, tell the ASR system which files we want to transcribe. You can find more details and download the `960h_scratch.pt` checkpoint -[[here]](https://github.com/pytorch/fairseq/blob/main/examples/wav2vec/README.md)). To run ASR, you would also need to -install KenLM, Flashlight decoder, and download the KenLM 4-gram English language model. - -```bash - python $FAIRSEQ_ROOT/examples/wav2vec/wav2vec_manifest.py \ - $UTS_OUTPUT_DOWNSAMPLE_CUT --valid-percent 0.0 --dest $MANIFEST_DIR --ext wav -``` -where `$UTS_OUTPUT_DOWNSAMPLE_CUT` speficies the directory with the preprocessed UTS outputs and `$MANIFEST_DIR` is the output directory. - -We will be running an out-of-the-box evaluation script which requires ground-truth transcripts to measure quality metrics. We are only -interested in the transcripts (and we don't have ground-truth outputs for when our ULM generated!), hence we will just generate -some dummy transcripts instead: -```bash -cp $FAIRSEQ_ROOT/examples/textless_nlp/gslm/asr_metrics/misc/dict.ltr.txt $MANIFEST_DIR -python $FAIRSEQ_ROOT/examples/textless_nlp/gslm/asr_metrics/misc/dummy_asr_data.py --tsv=$MANIFEST_DIR/train.tsv \ - --output-dir=$MANIFEST_DIR -``` - -Now we are ready for running ASR: -``` -mkdir -p asr -python $FAIRSEQ_ROOT/examples/speech_recognition/infer.py \ - $MANIFEST_DIR \ - --task audio_pretraining --nbest 1 --path 960h_scratch.pt \ - --gen-subset=train --results-path $PATH_TO_ASR_OUTPUT \ - --w2l-decoder kenlm --lm-model 4-gram.bin \ - --lexicon librispeech/lexicon_ltr.lst --word-score -1 \ - --sil-weight 0 --lm-weight 2 --criterion ctc --labels ltr --max-tokens 300000 --remove-bpe letter -``` -where `lexicon_ltr.lst` is the LibriSpeech lexicon and `$PATH_TO_ASR_OUTPUT` is the output directory (can be downloaded [[here]](https://dl.fbaipublicfiles.com/textless_nlp/gslm/eval_data/lexicon_ltr.lst)). - -## Evaluation metrics -We run evaluation on the 1_000 shortest sequences that are at least 6s long. To filter those from the ASR transcript, we additionally provide each metric script with the paths to the manifest and `ground_truth_continuation_*` files. - -### Perplexity (PPX) -To get a PPX metric estimate on an ASR transcript, you need to run the following command: -```bash -python ppx.py $PATH_TO_ASR_OUTPUT/hypo.word-960h_scratch.pt-train.txt --cut-tail\ - --manifest=$MANIFEST_DIR/train.tsv --prompts-description=data/ground_truth_continuation_dev.json -``` -where `--cut-tail` tells the script to ignore the last token on each line (ASR puts the sequence ID there). - -### Self- and Auto-BLEU -```bash -python self_bleu.py $PATH_TO_ASR_OUTPUT/hypo.word-960h_scratch.pt-train.txt --cut-tail \ - --manifest=$MANIFEST_DIR/train.tsv --prompts-description=data/ground_truth_continuation_dev.json -``` - -### Continuation-BLEU -```bash -python continuation_eval.py --asr-transcript $PATH_TO_ASR_OUTPUT/hypo.word-960h_scratch.pt-train.txt \ - --manifest=$MANIFEST_DIR/train.tsv --prompts-description=data/ground_truth_continuation_dev.json -``` - -### AUC -Based on the metrics calculated above, we can estimate the AUC of the perplexity/diversity trade-off. We provide an illustration in a [Colab notebook](https://colab.research.google.com/drive/1pVPfOVax_PU3MkYdHRSsa-SI8GBUldNt?usp=sharing). diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/metrics/asr_metrics/continuation_eval.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/metrics/asr_metrics/continuation_eval.py deleted file mode 100644 index f55b6cf003152332b6149e6da374741cf8263373..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/metrics/asr_metrics/continuation_eval.py +++ /dev/null @@ -1,100 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - - -import json -import warnings -from collections import defaultdict - -import numpy as np -from misc.bleu_utils import sentence_bleu - - -def get_args(): - import argparse - - parser = argparse.ArgumentParser("Tool to calculate Continuation-BLEU2") - parser.add_argument('--asr-transcript', type=str, - help='Path to the transcript file.') - parser.add_argument('--prompts-description', type=str, - help='Path to the ground-truth continuation') - parser.add_argument('--manifest', type=str, required=True) - parser.add_argument('--take-shortest', type=int, default=1000) - - args = parser.parse_args() - - return args - - -def main(): - # NLTK produces warnings - warnings.filterwarnings("ignore") - - args = get_args() - - with open(args.prompts_description, 'r') as fin: - original_continuations = json.loads(fin.read()) - - sequence2length = [(k, v[0]) for k, v in original_continuations.items()] - assert all(float(v) >= 6.0 for (_, v) in sequence2length) # 6 seconds - - sequence2length.sort(key=lambda x: x[1]) - to_take = set(v[0] for v in sequence2length[:args.take_shortest]) - - with open(args.manifest, 'r') as fin: - fin.readline() - - linenum2file = dict([ - (i, l.split("__")[0]) for (i, l) in enumerate(fin) - ]) - - max_files = max(linenum2file.keys()) - continuations = defaultdict(list) - - mean_length_after = 0 - n_examples = 0 - - with open(args.asr_transcript, 'r') as fin: - for line in fin: - n_examples += 1 - line = line.split() - sequence_id = int(line[-1].split('-')[1][:-1]) - - assert sequence_id <= max_files - - sequence_name = linenum2file[sequence_id] - - continuations[sequence_name].append(line[:-1]) - mean_length_after += len(line) - - mean_length_after /= n_examples - print(f'Mean length of continuations, in words: {mean_length_after}') - metric_values = [] - - mean_ground_truth_words = 0 - n_examples = 0 - n_candidates = 0 - - for k, candidates in continuations.items(): - if k not in to_take: - continue - - n_examples += 1 - - ground_truth = original_continuations[k][1].split() - n_candidates += len(candidates) - bleu = sentence_bleu(candidates, ground_truth, weights=( - 0.5, 0.5), no_length_penalty=True, averaging_mode="geometric") - mean_ground_truth_words += len(ground_truth) - - metric_values.append(bleu) - - n = len(metric_values) - print( - f'Median BLEU over {n} examples: {np.median(metric_values)} +- {np.std(metric_values) / np.sqrt(n)}') - - -if __name__ == '__main__': - main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/metrics/asr_metrics/misc/bleu_utils.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/metrics/asr_metrics/misc/bleu_utils.py deleted file mode 100644 index d2563a222256b906ec52caf5c0059d99f2e56823..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/metrics/asr_metrics/misc/bleu_utils.py +++ /dev/null @@ -1,172 +0,0 @@ -""" - -TODO: the code is take from Apache-2 Licensed NLTK: make sure we do this properly! - - -Copied over from nltk.tranlate.bleu_score. This code has two major changes: - - allows to turn off length/brevity penalty --- it has no sense for self-bleu, - - allows to use arithmetic instead of geometric mean -""" - -import math -import sys -import warnings -from collections import Counter -from fractions import Fraction - -from nltk.translate.bleu_score import ( - SmoothingFunction, - brevity_penalty, - closest_ref_length, - modified_precision, -) - - -def corpus_bleu( - list_of_references, - hypotheses, - weights=(0.25, 0.25, 0.25, 0.25), - smoothing_function=None, - auto_reweigh=False, - averaging_mode="geometric", - no_length_penalty=False -): - """ - Calculate a single corpus-level BLEU score (aka. system-level BLEU) for all - the hypotheses and their respective references. - - Instead of averaging the sentence level BLEU scores (i.e. marco-average - precision), the original BLEU metric (Papineni et al. 2002) accounts for - the micro-average precision (i.e. summing the numerators and denominators - for each hypothesis-reference(s) pairs before the division). - - >>> hyp1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'which', - ... 'ensures', 'that', 'the', 'military', 'always', - ... 'obeys', 'the', 'commands', 'of', 'the', 'party'] - >>> ref1a = ['It', 'is', 'a', 'guide', 'to', 'action', 'that', - ... 'ensures', 'that', 'the', 'military', 'will', 'forever', - ... 'heed', 'Party', 'commands'] - >>> ref1b = ['It', 'is', 'the', 'guiding', 'principle', 'which', - ... 'guarantees', 'the', 'military', 'forces', 'always', - ... 'being', 'under', 'the', 'command', 'of', 'the', 'Party'] - >>> ref1c = ['It', 'is', 'the', 'practical', 'guide', 'for', 'the', - ... 'army', 'always', 'to', 'heed', 'the', 'directions', - ... 'of', 'the', 'party'] - - >>> hyp2 = ['he', 'read', 'the', 'book', 'because', 'he', 'was', - ... 'interested', 'in', 'world', 'history'] - >>> ref2a = ['he', 'was', 'interested', 'in', 'world', 'history', - ... 'because', 'he', 'read', 'the', 'book'] - - >>> list_of_references = [[ref1a, ref1b, ref1c], [ref2a]] - >>> hypotheses = [hyp1, hyp2] - >>> corpus_bleu(list_of_references, hypotheses) # doctest: +ELLIPSIS - 0.5920... - - The example below show that corpus_bleu() is different from averaging - sentence_bleu() for hypotheses - - >>> score1 = sentence_bleu([ref1a, ref1b, ref1c], hyp1) - >>> score2 = sentence_bleu([ref2a], hyp2) - >>> (score1 + score2) / 2 # doctest: +ELLIPSIS - 0.6223... - - :param list_of_references: a corpus of lists of reference sentences, w.r.t. hypotheses - :type list_of_references: list(list(list(str))) - :param hypotheses: a list of hypothesis sentences - :type hypotheses: list(list(str)) - :param weights: weights for unigrams, bigrams, trigrams and so on - :type weights: list(float) - :param smoothing_function: - :type smoothing_function: SmoothingFunction - :param auto_reweigh: Option to re-normalize the weights uniformly. - :type auto_reweigh: bool - :return: The corpus-level BLEU score. - :rtype: float - """ - # Before proceeding to compute BLEU, perform sanity checks. - - p_numerators = Counter() # Key = ngram order, and value = no. of ngram matches. - p_denominators = Counter() # Key = ngram order, and value = no. of ngram in ref. - hyp_lengths, ref_lengths = 0, 0 - - assert len(list_of_references) == len(hypotheses), ( - "The number of hypotheses and their reference(s) should be the " "same " - ) - - # Iterate through each hypothesis and their corresponding references. - for references, hypothesis in zip(list_of_references, hypotheses): - # For each order of ngram, calculate the numerator and - # denominator for the corpus-level modified precision. - for i, _ in enumerate(weights, start=1): - p_i = modified_precision(references, hypothesis, i) - p_numerators[i] += p_i.numerator - p_denominators[i] += p_i.denominator - - # Calculate the hypothesis length and the closest reference length. - # Adds them to the corpus-level hypothesis and reference counts. - hyp_len = len(hypothesis) - hyp_lengths += hyp_len - ref_lengths += closest_ref_length(references, hyp_len) - - # Calculate corpus-level brevity penalty. - if no_length_penalty and averaging_mode == 'geometric': - bp = 1.0 - elif no_length_penalty and averaging_mode == 'arithmetic': - bp = 0.0 - else: - assert not no_length_penalty - assert averaging_mode != 'arithmetic', 'Not sure how to apply length penalty when aurithmetic mode' - bp = brevity_penalty(ref_lengths, hyp_lengths) - - # Uniformly re-weighting based on maximum hypothesis lengths if largest - # order of n-grams < 4 and weights is set at default. - if auto_reweigh: - if hyp_lengths < 4 and weights == (0.25, 0.25, 0.25, 0.25): - weights = (1 / hyp_lengths,) * hyp_lengths - - # Collects the various precision values for the different ngram orders. - p_n = [ - Fraction(p_numerators[i], p_denominators[i], _normalize=False) - for i, _ in enumerate(weights, start=1) - ] - - # Returns 0 if there's no matching n-grams - # We only need to check for p_numerators[1] == 0, since if there's - # no unigrams, there won't be any higher order ngrams. - if p_numerators[1] == 0: - return 0 - - # If there's no smoothing, set use method0 from SmoothinFunction class. - if not smoothing_function: - smoothing_function = SmoothingFunction().method0 - # Smoothen the modified precision. - # Note: smoothing_function() may convert values into floats; - # it tries to retain the Fraction object as much as the - # smoothing method allows. - p_n = smoothing_function( - p_n, references=references, hypothesis=hypothesis, hyp_len=hyp_lengths - ) - - if averaging_mode == "geometric": - s = (w_i * math.log(p_i) for w_i, p_i in zip(weights, p_n)) - s = bp * math.exp(math.fsum(s)) - elif averaging_mode == "arithmetic": - s = (w_i * p_i for w_i, p_i in zip(weights, p_n)) - s = math.fsum(s) - - return s - - -def sentence_bleu( - references, - hypothesis, - weights=(0.25, 0.25, 0.25, 0.25), - smoothing_function=None, - auto_reweigh=False, - averaging_mode="geometric", - no_length_penalty=False -): - return corpus_bleu( - [references], [hypothesis], weights, smoothing_function, auto_reweigh, averaging_mode, no_length_penalty - ) \ No newline at end of file diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/metrics/asr_metrics/misc/cut_as.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/metrics/asr_metrics/misc/cut_as.py deleted file mode 100644 index 4dec8c75e7ce3a89ca7d9d4016c1d404f8d6c0da..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/metrics/asr_metrics/misc/cut_as.py +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - - -import argparse -import json -import pathlib - -import torchaudio - - -def get_args(): - parser = argparse.ArgumentParser( - "Assuring generated audio have the same length as ground-truth audio") - parser.add_argument('--samples_dir', required=True, type=str) - parser.add_argument('--out_dir', required=True, type=str) - parser.add_argument('--prompts_description', required=True, type=str) - return parser.parse_args() - - -def cut(src, tgt, l): - x, sr = torchaudio.load(str(src)) - assert sr == 16_000 - - x = x.squeeze() - target_frames = int(l * sr) - - flag = 0 - if target_frames <= x.size(0): - x = x[:target_frames] - flag = 1 - else: - flag = 0 - torchaudio.save(str(tgt), x.unsqueeze(0), sr) - return flag - - -def main(): - args = get_args() - tgt_dir = pathlib.Path(args.out_dir) - tgt_dir.mkdir(exist_ok=True, parents=True) - - total_files, sufficiently_long = 0, 0 - - with open(args.prompts_description, 'r') as f: - description = json.loads(f.read()) - - for src_f in pathlib.Path(args.samples_dir).glob('*.wav'): - name_prompt = src_f.with_suffix('').name.split('__')[0] - - assert name_prompt in description, f'Cannot find {name_prompt}!' - - target_length = description[name_prompt][0] - tgt_f = tgt_dir / (src_f.name) - - is_long_enough = cut(src_f, tgt_f, target_length) - sufficiently_long += is_long_enough - if not is_long_enough: - print(f'{src_f} is not long enough') - - total_files += 1 - - print( - f'Total files: {total_files}; sufficiently long: {sufficiently_long}') - - -if __name__ == '__main__': - main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/metrics/asr_metrics/misc/dict.ltr.txt b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/metrics/asr_metrics/misc/dict.ltr.txt deleted file mode 100644 index 69929e1666c8182148d83ef4332e4c677bb90e5a..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/metrics/asr_metrics/misc/dict.ltr.txt +++ /dev/null @@ -1,28 +0,0 @@ -| 94802 -E 51860 -T 38431 -A 33152 -O 31495 -N 28855 -I 28794 -H 27187 -S 26071 -R 23546 -D 18289 -L 16308 -U 12400 -M 10685 -W 10317 -C 9844 -F 9062 -G 8924 -Y 8226 -P 6890 -B 6339 -V 3936 -K 3456 -' 1023 -X 636 -J 598 -Q 437 -Z 213 diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/metrics/asr_metrics/ppx.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/metrics/asr_metrics/ppx.py deleted file mode 100644 index 0253c6753e40372192264b0db5e1f2a3ee7cf2cf..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/metrics/asr_metrics/ppx.py +++ /dev/null @@ -1,123 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - - -import warnings - -import numpy as np -import torch - - -def get_target_sequences(manifest, ground_truth, to_take=1000): - import json - import pathlib - - with open(ground_truth, 'r') as fin: - original_continuations = json.loads(fin.read()) - - sequence2length = [(k, v[0]) for k, v in original_continuations.items()] - assert all(float(v) >= 6.0 for (_, v) in sequence2length) # 6 seconds - - sequence2length.sort(key=lambda x: x[1]) - to_take_sequences = set(v[0] for v in sequence2length[:to_take]) - to_take_ids = [] - - with open(manifest, 'r') as f: - f.readline() - - for i, line in enumerate(f.readlines()): - seq_id = line.split()[0] - seq_id = pathlib.Path(seq_id).name.split('__')[0] - - if seq_id in to_take_sequences: - to_take_ids.append(i) - - print(f'Took {len(to_take_ids)} ids') - return set(to_take_ids) - - -def get_args(): - import argparse - - parser = argparse.ArgumentParser("Evaluate PPX metric of a transcript.") - parser.add_argument('--asr-transcript', type=str, - help='Path to the transcript file.') - parser.add_argument('--cut-id', action='store_true', - help='Whether cut the first token (typically a seq id)') - parser.add_argument('--cut-tail', action='store_true', - help='Whether cut the last token (typically a speaker id)') - - parser.add_argument('--manifest', type=str, default=None) - parser.add_argument('--prompts-description', type=str, default=None) - - args = parser.parse_args() - - return args - - -def main(): - args = get_args() - - lm = torch.hub.load( - 'pytorch/fairseq', 'transformer_lm.wmt19.en', tokenizer='moses', bpe='fastbpe') - - lm.eval().cuda() # disable dropout - - if args.manifest is None and args.prompts_description is None: - target_ids = None - else: - target_ids = get_target_sequences( - args.manifest, args.prompts_description) - - with open(args.asr_transcript, 'r') as fin: - lines = fin.readlines() - - if target_ids is not None: - filtered = [] - for line in lines: - line_id = line.split()[-1] - line_id = int(line_id.split('-')[1][:-1]) - if line_id in target_ids: - filtered.append(line) - lines = filtered - else: - pass - - if args.cut_id: - lines = [' '.join(x.split()[1:]) for x in lines] - if args.cut_tail: - lines = [' '.join(x.split()[:-1]) for x in lines] - lines = [x.strip().lower() for x in lines] - - def get_logprob(sent): return \ - lm.score(sent)['positional_scores'].mean().neg().item() - - logprobs = [get_logprob(l) for l in lines] - - filtered = [x for x in logprobs if not np.isnan(x)] - if len(filtered) != len(logprobs): - warnings.warn("NaNs detected!") - logprobs = filtered - - perplexities = [np.exp(l) for l in logprobs] - - for name, stats in [('logprob', logprobs), ('perplexity', perplexities)]: - mean = np.mean(stats) - sem = np.std(stats) / np.sqrt(len(stats)) - - median = np.median(stats) - interval = list(np.percentile(stats, [10, 90])) - - mean, sem, median, percentile10, percentile90 = [ - round(x, 2) for x in [mean, sem, median] + interval] - - print(name) - print(f"\tMean {mean} +- {sem}") - print( - f"\tMedian {median}, 90% confidence interval {percentile10}...{percentile90}") - - -if __name__ == '__main__': - main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/metrics/asr_metrics/self_auto_bleu.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/metrics/asr_metrics/self_auto_bleu.py deleted file mode 100644 index 07c2de00b6057bce32e3155e4ab1fe8cf7f593e1..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/metrics/asr_metrics/self_auto_bleu.py +++ /dev/null @@ -1,202 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import warnings - -import nltk -import numpy as np -from misc.bleu_utils import sentence_bleu - - -def get_target_sequences(manifest, ground_truth, to_take=1000): - import json - import pathlib - - with open(ground_truth, 'r') as fin: - original_continuations = json.loads(fin.read()) - - sequence2length = [(k, v[0]) for k, v in original_continuations.items()] - assert all(float(v) >= 6.0 for (_, v) in sequence2length) # 6 seconds - - sequence2length.sort(key=lambda x: x[1]) - to_take_sequences = set(v[0] for v in sequence2length[:to_take]) - to_take_ids = [] - - with open(manifest, 'r') as f: - f.readline() - - for i, line in enumerate(f.readlines()): - seq_id = line.split()[0] - seq_id = pathlib.Path(seq_id).name.split('__')[0] - - if seq_id in to_take_sequences: - to_take_ids.append(i) - - print(f'Took {len(to_take_ids)} ids') - return set(to_take_ids) - - -def get_args(): - import argparse - - parser = argparse.ArgumentParser() - parser.add_argument('--asr-transcript', type=str, - help='Path to the transcript file.') - - parser.add_argument('--manifest', required=True) - parser.add_argument('--prompts-description', required=True) - - parser.add_argument('--cut-id', action='store_true', - help='Whether cut the first token (typically a seq id)') - parser.add_argument('--cut-tail', action='store_true', - help='Whether cut the last token (typically a speaker id)') - parser.add_argument('--debug', action='store_true') - - args = parser.parse_args() - - return args - - -def get_self_bleu(utterances, averaging_mode, weights): - self_bleu = [] - - for i in range(len(utterances)): - hypo = utterances[i] - rest = utterances[:i] + utterances[i+1:] - - self_bleu.append(sentence_bleu(rest, hypo, weights, - no_length_penalty=True, averaging_mode=averaging_mode)) - - return self_bleu - - -def get_self_bleu2_arithmetic(utterances): - weights = (0.5, 0.5) # equal weight for unigrams and bigrams - return get_self_bleu(utterances, averaging_mode='arithmetic', weights=weights) - - -def get_self_bleu2_geometric(utterances): - weights = (0.5, 0.5) - return get_self_bleu(utterances, averaging_mode='geometric', weights=weights) - - -def get_auto_bleu2_arithmetic(utterances): - weights = (0.5, 0.5) - return [auto_bleu(u, mean_mode='arithmetic', weights=weights) for u in utterances] - - -def get_auto_bleu2_geometric(utterances): - weights = (0.5, 0.5) - return [auto_bleu(u, mean_mode='geometric', weights=weights) for u in utterances] - - -def get_auto_bleu3_geometric(utterances): - weights = (1./3, 1./3, 1./3) - return [auto_bleu(u, mean_mode='geometric', weights=weights) for u in utterances] - - -def get_auto_bleu3_arithmetic(utterances): - weights = (1./3, 1./3, 1./3) - return [auto_bleu(u, mean_mode='arithmetic', weights=weights) for u in utterances] - - -def get_self_bleu3_arithmetic(utterances): - weights = (1./3, 1./3, 1./3) - return get_self_bleu(utterances, averaging_mode='arithmetic', weights=weights) - - -def get_self_bleu3_geometric(utterances): - weights = (1./3, 1./3, 1./3) - return get_self_bleu(utterances, averaging_mode='geometric', weights=weights) - - -def auto_bleu(sentence, weights, mean_mode='arithmetic'): - if len(sentence) <= 1: - return 0 - - N = len(weights) - - bleu_n = np.zeros([N]) - for n in range(N): - targ_ngrams = list(nltk.ngrams(sentence, n+1)) - for p in range(len(targ_ngrams)): - left = sentence[:p] - right = sentence[(p+n+1):] - rest_ngrams = list(nltk.ngrams(left, n+1)) + \ - list(nltk.ngrams(right, n+1)) - # compute the nb of matching ngrams - bleu_n[n] += targ_ngrams[p] in rest_ngrams - bleu_n[n] /= len(targ_ngrams) # average them to get a proportion - - weights = np.array(weights) - if mean_mode == 'arithmetic': - return (bleu_n * weights).sum() - elif mean_mode == 'geometric': - return (bleu_n ** weights).prod() - else: - raise ValueError(f'Unknown agggregation mode {mean_mode}') - - -def main(): - from multiprocessing import Pool - - args = get_args() - target_ids = get_target_sequences(args.manifest, args.prompts_description) - - with open(args.asr_transcript, 'r') as fin: - lines = fin.readlines() - - terms = [x.strip().split() for x in lines] - filtered = [] - for term in terms: - line_id = int(term[-1].split('-')[1][:-1]) - if line_id in target_ids: - filtered.append(term) - terms = filtered - - if args.cut_id: - terms = [x[1:] for x in terms] - if args.cut_tail: - terms = [x[:-1] for x in terms] - - if args.debug: - terms = terms[:10] - - tasks = [ - ('Self-BLEU2-arithmetic', get_self_bleu2_arithmetic), - ('Self-BLEU2-geometric', get_self_bleu2_geometric), - ('Auto-BLEU2-arithmetic', get_auto_bleu2_arithmetic), - ('Auto-BLEU2-geometric', get_auto_bleu2_geometric), - - ('Self-BLEU3-arithmetic', get_self_bleu3_arithmetic), - ('Self-BLEU3-geometric', get_self_bleu3_geometric), - ('Auto-BLEU3-arithmetic', get_auto_bleu3_arithmetic), - ('Auto-BLEU3-geometric', get_auto_bleu3_geometric), - ] - - n_processes = min(16, len(tasks)) - with Pool(n_processes) as pool: - metrics = pool.map(run_f, [(t[1], terms) for t in tasks]) - - for (metric_name, _), metric in zip(tasks, metrics): - metric, sem = np.mean(metric), np.std(metric) / np.sqrt(len(metric)) - - metric, sem = [ - round(100 * x, 2) for x in [metric, sem] - ] - - print(f'{metric_name} {metric} +- {sem}') - - -def run_f(task_params): - f, terms = task_params - return f(terms) - - -if __name__ == '__main__': - # NLTK produces warnings - warnings.filterwarnings("ignore") - - main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/speech2unit/README.md b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/speech2unit/README.md deleted file mode 100644 index 9dff9d33ac6cc1a7e2c6c7c653d1265123174abd..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/speech2unit/README.md +++ /dev/null @@ -1,68 +0,0 @@ -# Speech to Unit Model (speech2unit) - -## Acoustic Model -For quantizing speech we learn a K-means clustering over acoustic representations for which we either use Log-Mel Filterbank or pretrained acoustic representation models. For using pretrained models, please download from their respective locations linked below. -* [Modified CPC](https://dl.fbaipublicfiles.com/textless_nlp/gslm/cpc/cpc_big_ll6kh_top_ctc.pt) -* [HuBERT-Base](https://dl.fbaipublicfiles.com/hubert/hubert_base_ls960.pt) -* [Wav2Vec 2.0-Base](https://dl.fbaipublicfiles.com/fairseq/wav2vec/wav2vec_vox_new.pt) - -## Quantization Model -You can download pretrained quantized model from the list below. - -K-Means Model | Download Link -|-|- -Log Mel Filterbank + KM50 | [download](https://dl.fbaipublicfiles.com/textless_nlp/gslm/logmel/km50/km.bin) -Log Mel Filterbank + KM100 | [download](https://dl.fbaipublicfiles.com/textless_nlp/gslm/logmel/km100/km.bin) -Log Mel Filterbank + KM200 | [download](https://dl.fbaipublicfiles.com/textless_nlp/gslm/logmel/km200/km.bin) -Modified CPC + KM50 | [download](https://dl.fbaipublicfiles.com/textless_nlp/gslm/cpc/km50/km.bin) -Modified CPC + KM100 | [download](https://dl.fbaipublicfiles.com/textless_nlp/gslm/cpc/km100/km.bin) -Modified CPC + KM200 | [download](https://dl.fbaipublicfiles.com/textless_nlp/gslm/cpc/km200/km.bin) -HuBERT Base + KM50 | [download](https://dl.fbaipublicfiles.com/textless_nlp/gslm/hubert/km50/km.bin) -HuBERT Base + KM100 | [download](https://dl.fbaipublicfiles.com/textless_nlp/gslm/hubert/km100/km.bin) -HuBERT Base + KM200 | [download](https://dl.fbaipublicfiles.com/textless_nlp/gslm/hubert/km200/km.bin) -wav2vec 2.0 Large + KM50 | [download](https://dl.fbaipublicfiles.com/textless_nlp/gslm/w2v2/km50/km.bin) -wav2vec 2.0 Large + KM100 | [download](https://dl.fbaipublicfiles.com/textless_nlp/gslm/w2v2/km100/km.bin) -wav2vec 2.0 Large + KM200 | [download](https://dl.fbaipublicfiles.com/textless_nlp/gslm/w2v2/km200/km.bin) - -### Quantization -For quantizing speech with a given acoustic representation, please follow the steps below. -1. Learn K-means clustering model -``` -N_CLUSTERS= -TYPE= -CKPT_PATH= -LAYER= -MANIFEST= -KM_MODEL_PATH= - -PYTHONPATH=. python examples/textless_nlp/gslm/speech2unit/clustering/cluster_kmeans.py \ - --num_clusters $N_CLUSTERS \ - --feature_type $TYPE \ - --checkpoint_path $CKPT_PATH \ - --layer $LAYER \ - --manifest_path $MANIFEST \ - --out_kmeans_model_path $KM_MODEL_PATH -``` -2. Quantize using the learned clusters -``` -MANIFEST= -OUT_QUANTIZED_FILE= - -python examples/textless_nlp/gslm/speech2unit/clustering/quantize_with_kmeans.py \ - --feature_type $TYPE \ - --kmeans_model_path $KM_MODEL_PATH \ - --acoustic_model_path $CKPT_PATH \ - --layer $LAYER \ - --manifest_path $MANIFEST \ - --out_quantized_file_path $OUT_QUANTIZED_FILE \ - --extension ".flac" -``` - -Note about the manifest file is a file with paths and length of input audio files. The format of the file is as follows: -``` - -\t -\t -... -``` - diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/speech2unit/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/speech2unit/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/speech2unit/clustering/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/speech2unit/clustering/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/speech2unit/clustering/cluster_kmeans.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/speech2unit/clustering/cluster_kmeans.py deleted file mode 100644 index eff0f60c1d56b429a28d0b89487e43bff9749087..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/speech2unit/clustering/cluster_kmeans.py +++ /dev/null @@ -1,211 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import logging -import os -import time - -import joblib -import numpy as np -from examples.textless_nlp.gslm.speech2unit.pretrained.utils import ( - get_and_dump_features, - get_features, -) -from sklearn.cluster import MiniBatchKMeans - - -def get_logger(): - log_format = "[%(asctime)s] [%(levelname)s]: %(message)s" - logging.basicConfig(format=log_format, level=logging.INFO) - logger = logging.getLogger(__name__) - return logger - - -def get_parser(): - parser = argparse.ArgumentParser( - description="Learn K-means clustering over acoustic features." - ) - - # Features arguments - parser.add_argument( - "--in_features_path", type=str, default=None, help="Features file path" - ) - parser.add_argument( - "--feature_type", - type=str, - choices=["logmel", "hubert", "w2v2", "cpc"], - default=None, - help="Acoustic feature type", - ) - parser.add_argument( - "--manifest_path", - type=str, - default=None, - help="Manifest file containing the root dir and file names", - ) - parser.add_argument( - "--out_features_path", - type=str, - default=None, - help="Features file path to write to", - ) - parser.add_argument( - "--checkpoint_path", - type=str, - help="Pretrained acoustic model checkpoint", - ) - parser.add_argument( - "--layer", - type=int, - help="The layer of the pretrained model to extract features from", - default=-1, - ) - parser.add_argument( - "--sample_pct", - type=float, - help="Percent data to use for K-means training", - default=0.1, - ) - - # K-means arguments - parser.add_argument( - "--num_clusters", type=int, help="Nubmer of clusters", default=50 - ) - parser.add_argument("--init", default="k-means++") - parser.add_argument( - "--max_iter", - type=int, - help="Maximum number of iterations for K-means training", - default=150, - ) - parser.add_argument( - "--batch_size", - type=int, - help="Batch size for K-means training", - default=10000, - ) - parser.add_argument("--tol", default=0.0, type=float) - parser.add_argument("--max_no_improvement", default=100, type=int) - parser.add_argument("--n_init", default=20, type=int) - parser.add_argument("--reassignment_ratio", default=0.5, type=float) - parser.add_argument( - "--out_kmeans_model_path", - type=str, - required=True, - help="Path to save K-means model", - ) - - # Leftovers - parser.add_argument( - "--seed", - type=int, - help="Random seed to use for K-means training", - default=1369, - ) - - return parser - - -def get_kmeans_model( - n_clusters, - init, - max_iter, - batch_size, - tol, - max_no_improvement, - n_init, - reassignment_ratio, - random_state, -): - return MiniBatchKMeans( - n_clusters=n_clusters, - init=init, - max_iter=max_iter, - batch_size=batch_size, - tol=tol, - max_no_improvement=max_no_improvement, - n_init=n_init, - reassignment_ratio=reassignment_ratio, - random_state=random_state, - verbose=1, - compute_labels=True, - init_size=None, - ) - - -def train_kmeans(kmeans_model, features_batch): - start_time = time.time() - kmeans_model.fit(features_batch) - time_taken = round((time.time() - start_time) // 60, 2) - return kmeans_model, time_taken - - -def main(args, logger): - # Features loading/extraction for K-means - if args.in_features_path: - # Feature loading - logger.info(f"Loading features from {args.in_features_path}...") - features_batch = np.load(args.in_features_path, allow_pickle=True) - else: - # Feature extraction - logger.info(f"Extracting {args.feature_type} acoustic features...") - features_batch = ( - get_features( - feature_type=args.feature_type, - checkpoint_path=args.checkpoint_path, - layer=args.layer, - manifest_path=args.manifest_path, - sample_pct=args.sample_pct, - flatten=True, - ) - if not args.out_features_path - else get_and_dump_features( - feature_type=args.feature_type, - checkpoint_path=args.checkpoint_path, - layer=args.layer, - manifest_path=args.manifest_path, - sample_pct=args.sample_pct, - flatten=True, - out_features_path=args.out_features_path, - ) - ) - if args.out_features_path: - logger.info( - f"Saved extracted features at {args.out_features_path}" - ) - logger.info(f"Features shape = {features_batch.shape}\n") - - # Learn and save K-means model - kmeans_model = get_kmeans_model( - n_clusters=args.num_clusters, - init=args.init, - max_iter=args.max_iter, - batch_size=args.batch_size, - tol=args.tol, - max_no_improvement=args.max_no_improvement, - n_init=args.n_init, - reassignment_ratio=args.reassignment_ratio, - random_state=args.seed, - ) - logger.info("Starting k-means training...") - kmeans_model, time_taken = train_kmeans( - kmeans_model=kmeans_model, features_batch=features_batch - ) - logger.info(f"...done k-means training in {time_taken} minutes") - inertia = -kmeans_model.score(features_batch) / len(features_batch) - logger.info(f"Total intertia: {round(inertia, 2)}\n") - - logger.info(f"Saving k-means model to {args.out_kmeans_model_path}") - os.makedirs(os.path.dirname(args.out_kmeans_model_path), exist_ok=True) - joblib.dump(kmeans_model, open(args.out_kmeans_model_path, "wb")) - - -if __name__ == "__main__": - parser = get_parser() - args = parser.parse_args() - logger = get_logger() - logger.info(args) - main(args, logger) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/speech2unit/clustering/dump_feats.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/speech2unit/clustering/dump_feats.py deleted file mode 100644 index 031567c6d85d16b5236053abf008b7cabccb4673..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/speech2unit/clustering/dump_feats.py +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import logging - -from examples.textless_nlp.gslm.speech2unit.pretrained.utils import ( - get_and_dump_features, -) - - -def get_parser(): - parser = argparse.ArgumentParser( - description="Compute and dump log mel fbank features." - ) - parser.add_argument( - "--feature_type", - type=str, - choices=["logmel", "hubert", "w2v2", "cpc"], - default=None, - help="Acoustic feature type", - ) - parser.add_argument( - "--manifest_path", - type=str, - default=None, - help="Manifest file containing the root dir and file names", - ) - parser.add_argument( - "--out_features_path", - type=str, - default=None, - help="Features file path to write to", - ) - parser.add_argument( - "--checkpoint_path", - type=str, - help="Pretrained acoustic model checkpoint", - ) - parser.add_argument( - "--layer", - type=int, - help="The layer of the pretrained model to extract features from", - default=-1, - ) - parser.add_argument( - "--sample_pct", - type=float, - help="Percent data to use for K-means training", - default=0.1, - ) - parser.add_argument( - "--out_features_path", - type=str, - help="Path to save log mel fbank features", - ) - return parser - - -def get_logger(): - log_format = "[%(asctime)s] [%(levelname)s]: %(message)s" - logging.basicConfig(format=log_format, level=logging.INFO) - logger = logging.getLogger(__name__) - return logger - - -if __name__ == "__main__": - """ - Example command: - python ~/speechbot/clustering/dump_logmelfank_feats.py \ - --manifest_path /checkpoint/kushall/data/LJSpeech-1.1/asr_input_wavs_16k/train.tsv - --out_features_path /checkpoint/kushall/experiments/speechbot/logmelfbank/features/ljspeech/train.npy - """ - parser = get_parser() - args = parser.parse_args() - logger = get_logger() - logger.info(args) - - logger.info(f"Extracting {args.feature_type} acoustic features...") - get_and_dump_features( - feature_type=args.feature_type, - checkpoint_path=args.checkpoint_path, - layer=args.layer, - manifest_path=args.manifest_path, - sample_pct=args.sample_pct, - flatten=True, - out_features_path=args.out_features_path, - ) - logger.info(f"Saved extracted features at {args.out_features_path}") diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/speech2unit/clustering/quantize_with_kmeans.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/speech2unit/clustering/quantize_with_kmeans.py deleted file mode 100644 index e0b7e001609571540458acb6b2dd1db60e696c3c..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/speech2unit/clustering/quantize_with_kmeans.py +++ /dev/null @@ -1,124 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import logging -import os - -import joblib -import numpy as np -from examples.textless_nlp.gslm.speech2unit.clustering.utils import ( - get_audio_files, -) -from examples.textless_nlp.gslm.speech2unit.pretrained.utils import ( - get_features, -) - - -def get_logger(): - log_format = "[%(asctime)s] [%(levelname)s]: %(message)s" - logging.basicConfig(format=log_format, level=logging.INFO) - logger = logging.getLogger(__name__) - return logger - - -def get_parser(): - parser = argparse.ArgumentParser( - description="Quantize using K-means clustering over acoustic features." - ) - parser.add_argument( - "--feature_type", - type=str, - choices=["logmel", "hubert", "w2v2", "cpc"], - default=None, - required=True, - help="Acoustic feature type", - ) - parser.add_argument( - "--acoustic_model_path", - type=str, - help="Pretrained acoustic model checkpoint" - ) - parser.add_argument( - "--layer", - type=int, - help="The layer of the pretrained model to extract features from", - default=-1, - ) - parser.add_argument( - "--kmeans_model_path", - type=str, - required=True, - help="K-means model file path to use for inference", - ) - parser.add_argument( - "--features_path", - type=str, - default=None, - help="Features file path. You don't need to enter acoustic model details if you have dumped features", - ) - parser.add_argument( - "--manifest_path", - type=str, - default=None, - help="Manifest file containing the root dir and file names", - ) - parser.add_argument( - "--out_quantized_file_path", - required=True, - type=str, - help="File path of quantized output.", - ) - parser.add_argument( - "--extension", type=str, default=".flac", help="Features file path" - ) - return parser - - -def main(args, logger): - # Feature extraction - if args.features_path is not None: - logger.info(f"Loading acoustic features from {args.features_path}...") - features_batch = np.load(args.features_path) - else: - logger.info(f"Extracting {args.feature_type} acoustic features...") - features_batch = get_features( - feature_type=args.feature_type, - checkpoint_path=args.acoustic_model_path, - layer=args.layer, - manifest_path=args.manifest_path, - sample_pct=1.0, - flatten=False, - ) - logger.info( - f"Features extracted for {len(features_batch)} utterances.\n" - ) - logger.info( - f"Dimensionality of representation = {features_batch[0].shape[1]}" - ) - - # K-means model - logger.info(f"Loading K-means model from {args.kmeans_model_path} ...") - kmeans_model = joblib.load(open(args.kmeans_model_path, "rb")) - kmeans_model.verbose = False - - _, fnames, _ = get_audio_files(args.manifest_path) - - os.makedirs(os.path.dirname(args.out_quantized_file_path), exist_ok=True) - print(f"Writing quantized predictions to {args.out_quantized_file_path}") - with open(args.out_quantized_file_path, "w") as fout: - for i, feats in enumerate(features_batch): - pred = kmeans_model.predict(feats) - pred_str = " ".join(str(p) for p in pred) - base_fname = os.path.basename(fnames[i]).rstrip(args.extension) - fout.write(f"{base_fname}|{pred_str}\n") - - -if __name__ == "__main__": - parser = get_parser() - args = parser.parse_args() - logger = get_logger() - logger.info(args) - main(args, logger) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/speech2unit/clustering/utils.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/speech2unit/clustering/utils.py deleted file mode 100644 index cf08d1fe4b470477b724aa8d770d91c0cac35a0e..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/speech2unit/clustering/utils.py +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from typing import List, Tuple - - -def get_audio_files(manifest_path: str) -> Tuple[str, List[str], List[int]]: - fnames, sizes = [], [] - with open(manifest_path, "r") as f: - root_dir = f.readline().strip() - for line in f: - items = line.strip().split("\t") - assert ( - len(items) == 2 - ), f"File must have two columns separated by tab. Got {line}" - fnames.append(items[0]) - sizes.append(int(items[1])) - return root_dir, fnames, sizes diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/speech2unit/pretrained/cpc_feature_reader.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/speech2unit/pretrained/cpc_feature_reader.py deleted file mode 100644 index c613f52d3c3de43a048849a231a9a34e2a883486..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/speech2unit/pretrained/cpc_feature_reader.py +++ /dev/null @@ -1,192 +0,0 @@ -import soundfile as sf -import torch -import torch.nn as nn -import torch.nn.functional as F - - -class CpcFeatureReader: - """ - Wrapper class to run inference on CPC model. - Helps extract features for a given audio file. - """ - - def __init__( - self, - checkpoint_path, - layer, - use_encoder_layer=False, - norm_features=False, - sample_rate=16000, - max_chunk=64000, - ): - self.model = load_cpc_model(checkpoint_path, layer).eval().cuda() - self.sample_rate = sample_rate - self.max_chunk = max_chunk - self.norm_features = norm_features - self.use_encoder_layer = use_encoder_layer - - def read_audio(self, path, ref_len=None): - wav, sr = sf.read(path) - if wav.ndim == 2: - wav = wav.mean(-1) - assert wav.ndim == 1, wav.ndim - assert sr == self.sample_rate, sr - if ref_len is not None and abs(ref_len - len(wav)) > 160: - print(f"ref {ref_len} != read {len(wav)} ({path})") - return wav - - def get_feats(self, file_path, ref_len=None): - x = self.read_audio(file_path, ref_len) - # Inspired from CPC_audio feature_loader.py - with torch.no_grad(): - x = torch.from_numpy(x).float().cuda() - x = x.view(1, 1, -1) - size = x.size(2) - feat = [] - start = 0 - while start < size: - if start + self.max_chunk > size: - break - x_chunk = x[..., start : start + self.max_chunk] - feat_chunk = self.model.extract_features( - source=x_chunk, - get_encoded=self.use_encoder_layer, - norm_output=self.norm_features, - ) - feat.append(feat_chunk) - start += self.max_chunk - - if start < size: - x_chunk = x[:, -self.max_chunk :] - feat_chunk = self.model.extract_features( - source=x_chunk, - get_encoded=self.use_encoder_layer, - norm_output=self.norm_features, - ) - df = x_chunk.size(2) // feat_chunk.size(1) - delta = (size - start) // df - feat.append(feat_chunk[:, -delta:]) - return torch.cat(feat, 1).squeeze(0) - - -def load_cpc_model(checkpoint_path, layer=None): - state_dict = torch.load(checkpoint_path) - weights = state_dict["weights"] - config = state_dict["config"] - if layer is not None: - config["nLevelsGRU"] = layer - - encoder = CPCEncoder(config["hiddenEncoder"]) - ar_net = CPCAR( - config["hiddenEncoder"], config["hiddenGar"], False, config["nLevelsGRU"] - ) - - model = CPCModel(encoder, ar_net) - model.load_state_dict(weights, strict=False) - model.config = config - - return model - - -class ChannelNorm(nn.Module): - def __init__(self, num_features, epsilon=1e-05, affine=True): - super(ChannelNorm, self).__init__() - if affine: - self.weight = nn.parameter.Parameter(torch.Tensor(1, num_features, 1)) - self.bias = nn.parameter.Parameter(torch.Tensor(1, num_features, 1)) - else: - self.weight = None - self.bias = None - self.epsilon = epsilon - self.p = 0 - self.affine = affine - self.reset_parameters() - - def reset_parameters(self): - if self.affine: - torch.nn.init.ones_(self.weight) - torch.nn.init.zeros_(self.bias) - - def forward(self, x): - cum_mean = x.mean(dim=1, keepdim=True) - cum_var = x.var(dim=1, keepdim=True) - x = (x - cum_mean) * torch.rsqrt(cum_var + self.epsilon) - if self.weight is not None: - x = x * self.weight + self.bias - return x - - -class CPCEncoder(nn.Module): - def __init__(self, hidden_dim=512): - super(CPCEncoder, self).__init__() - self.conv0 = nn.Conv1d(1, hidden_dim, 10, stride=5, padding=3) - self.batchNorm0 = ChannelNorm(hidden_dim) - self.conv1 = nn.Conv1d(hidden_dim, hidden_dim, 8, stride=4, padding=2) - self.batchNorm1 = ChannelNorm(hidden_dim) - self.conv2 = nn.Conv1d(hidden_dim, hidden_dim, 4, stride=2, padding=1) - self.batchNorm2 = ChannelNorm(hidden_dim) - self.conv3 = nn.Conv1d(hidden_dim, hidden_dim, 4, stride=2, padding=1) - self.batchNorm3 = ChannelNorm(hidden_dim) - self.conv4 = nn.Conv1d(hidden_dim, hidden_dim, 4, stride=2, padding=1) - self.batchNorm4 = ChannelNorm(hidden_dim) - self.DOWNSAMPLING = 160 - - def get_output_dim(self): - return self.conv4.out_channels - - def forward(self, x): - x = F.relu(self.batchNorm0(self.conv0(x))) - x = F.relu(self.batchNorm1(self.conv1(x))) - x = F.relu(self.batchNorm2(self.conv2(x))) - x = F.relu(self.batchNorm3(self.conv3(x))) - x = F.relu(self.batchNorm4(self.conv4(x))) - return x - - -class CPCAR(nn.Module): - def __init__(self, dim_encoded, dim_output, keep_hidden, num_layers): - super(CPCAR, self).__init__() - self.baseNet = nn.LSTM( - dim_encoded, dim_output, num_layers=num_layers, batch_first=True - ) - self.hidden = None - self.keep_hidden = keep_hidden - - def get_output_dim(self): - return self.baseNet.hidden_size - - def forward(self, x): - try: - self.baseNet.flatten_parameters() - except RuntimeError: - pass - x, h = self.baseNet(x, self.hidden) - if self.keep_hidden: - if isinstance(h, tuple): - self.hidden = tuple(x.detach() for x in h) - else: - self.hidden = h.detach() - return x - - -class CPCModel(nn.Module): - def __init__(self, encoder, ar_net): - super(CPCModel, self).__init__() - self.gEncoder = encoder - self.gAR = ar_net - self.config = None - - def forward(self, x, label): - encoded = self.gEncoder(x).permute(0, 2, 1) - cpc_feature = self.gAR(encoded) - return cpc_feature, encoded, label - - def extract_features(self, source, get_encoded=False, norm_output=False): - cpc_feature, encoded, _ = self.forward(source, None) - if get_encoded: - cpc_feature = encoded - if norm_output: - mean = cpc_feature.mean(dim=1, keepdim=True) - var = cpc_feature.var(dim=1, keepdim=True) - cpc_feature = (cpc_feature - mean) / torch.sqrt(var + 1e-08) - return cpc_feature diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/speech2unit/pretrained/hubert_feature_reader.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/speech2unit/pretrained/hubert_feature_reader.py deleted file mode 100644 index d416a561ae99013e4b5e296adcc61ad41644bf77..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/speech2unit/pretrained/hubert_feature_reader.py +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import fairseq -import soundfile as sf -import torch -import torch.nn.functional as F - - -class HubertFeatureReader: - """ - Wrapper class to run inference on HuBERT model. - Helps extract features for a given audio file. - """ - - def __init__(self, checkpoint_path, layer, max_chunk=1600000): - ( - model, - cfg, - task, - ) = fairseq.checkpoint_utils.load_model_ensemble_and_task( - [checkpoint_path] - ) - self.model = model[0].eval().cuda() - self.task = task - self.layer = layer - self.max_chunk = max_chunk - - def read_audio(self, path, ref_len=None): - wav, sr = sf.read(path) - if wav.ndim == 2: - wav = wav.mean(-1) - assert wav.ndim == 1, wav.ndim - assert sr == self.task.cfg.sample_rate, sr - if ref_len is not None and abs(ref_len - len(wav)) > 160: - print(f"ref {ref_len} != read {len(wav)} ({path})") - return wav - - def get_feats(self, file_path, ref_len=None): - x = self.read_audio(file_path, ref_len) - with torch.no_grad(): - x = torch.from_numpy(x).float().cuda() - if self.task.cfg.normalize: - x = F.layer_norm(x, x.shape) - x = x.view(1, -1) - - feat = [] - for start in range(0, x.size(1), self.max_chunk): - x_chunk = x[:, start: start + self.max_chunk] - feat_chunk, _ = self.model.extract_features( - source=x_chunk, - padding_mask=None, - mask=False, - output_layer=self.layer, - ) - feat.append(feat_chunk) - return torch.cat(feat, 1).squeeze(0) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/speech2unit/pretrained/logmel_feature_reader.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/speech2unit/pretrained/logmel_feature_reader.py deleted file mode 100644 index 106f50247622deca688b223f1ad63275d5b65e58..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/speech2unit/pretrained/logmel_feature_reader.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import soundfile as sf -import torch -import torchaudio.compliance.kaldi as kaldi - - -class LogMelFeatureReader: - """ - Wrapper class to run inference on HuBERT model. - Helps extract features for a given audio file. - """ - - def __init__(self, *args, **kwargs): - self.num_mel_bins = kwargs.get("num_mel_bins", 80) - self.frame_length = kwargs.get("frame_length", 25.0) - - def get_feats(self, file_path): - wav, sr = sf.read(file_path) - feats = torch.from_numpy(wav).float() - feats = kaldi.fbank( - feats.unsqueeze(0), - num_mel_bins=self.num_mel_bins, - frame_length=self.frame_length, - sample_frequency=sr, - ) - return feats diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/speech2unit/pretrained/utils.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/speech2unit/pretrained/utils.py deleted file mode 100644 index c591392153c9f465afb500d1f1695afab0c5e7b9..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/speech2unit/pretrained/utils.py +++ /dev/null @@ -1,126 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import gc -import os -import random -import shutil - -import numpy as np -import torch -import tqdm -from examples.textless_nlp.gslm.speech2unit.pretrained.cpc_feature_reader import ( - CpcFeatureReader, -) -from examples.textless_nlp.gslm.speech2unit.pretrained.hubert_feature_reader import ( - HubertFeatureReader, -) -from examples.textless_nlp.gslm.speech2unit.pretrained.logmel_feature_reader import ( - LogMelFeatureReader, -) -from examples.textless_nlp.gslm.speech2unit.pretrained.w2v2_feature_reader import ( - Wav2VecFeatureReader, -) - - -def get_feature_reader(feature_type): - if feature_type == "logmel": - return LogMelFeatureReader - elif feature_type == "hubert": - return HubertFeatureReader - elif feature_type == "w2v2": - return Wav2VecFeatureReader - elif feature_type == "cpc": - return CpcFeatureReader - else: - raise NotImplementedError(f"{feature_type} is not supported.") - - -def get_feature_iterator( - feature_type, checkpoint_path, layer, manifest_path, sample_pct -): - feature_reader_cls = get_feature_reader(feature_type) - with open(manifest_path, "r") as fp: - lines = fp.read().split("\n") - root = lines.pop(0).strip() - file_path_list = [ - os.path.join(root, line.split("\t")[0]) - for line in lines - if len(line) > 0 - ] - if sample_pct < 1.0: - file_path_list = random.sample( - file_path_list, int(sample_pct * len(file_path_list)) - ) - num_files = len(file_path_list) - reader = feature_reader_cls( - checkpoint_path=checkpoint_path, layer=layer - ) - - def iterate(): - for file_path in file_path_list: - feats = reader.get_feats(file_path) - yield feats.cpu().numpy() - - return iterate, num_files - - -def get_features( - feature_type, checkpoint_path, layer, manifest_path, sample_pct, flatten -): - generator, num_files = get_feature_iterator( - feature_type=feature_type, - checkpoint_path=checkpoint_path, - layer=layer, - manifest_path=manifest_path, - sample_pct=sample_pct, - ) - iterator = generator() - - features_list = [] - for features in tqdm.tqdm(iterator, total=num_files): - features_list.append(features) - - # Explicit clean up - del iterator - del generator - gc.collect() - torch.cuda.empty_cache() - - if flatten: - return np.concatenate(features_list) - - return features_list - - -def get_and_dump_features( - feature_type, - checkpoint_path, - layer, - manifest_path, - sample_pct, - flatten, - out_features_path, -): - # Feature extraction - features_batch = get_features( - feature_type=feature_type, - checkpoint_path=checkpoint_path, - layer=layer, - manifest_path=manifest_path, - sample_pct=sample_pct, - flatten=flatten, - ) - - # Save features - out_dir_path = os.path.dirname(out_features_path) - os.makedirs(out_dir_path, exist_ok=True) - shutil.copyfile( - manifest_path, - os.path.join(out_dir_path, os.path.basename(manifest_path)), - ) - np.save(out_features_path, features_batch) - - return features_batch diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/speech2unit/pretrained/w2v2_feature_reader.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/speech2unit/pretrained/w2v2_feature_reader.py deleted file mode 100644 index e9751d9b468d3999cf6c4ed78e22890e522ab8b3..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/speech2unit/pretrained/w2v2_feature_reader.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import fairseq -import soundfile as sf -import torch - - -class Wav2VecFeatureReader: - """ - Wrapper class to run inference on Wav2Vec 2.0 model. - Helps extract features for a given audio file. - """ - - def __init__(self, checkpoint_path, layer): - state = fairseq.checkpoint_utils.load_checkpoint_to_cpu( - checkpoint_path - ) - - w2v_args = state["args"] - self.task = fairseq.tasks.setup_task(w2v_args) - model = self.task.build_model(w2v_args) - model.load_state_dict(state["model"], strict=True) - model.eval() - model.cuda() - self.model = model - self.layer = layer - - def read_audio(self, fname): - wav, sr = sf.read(fname) - if wav.ndim == 2: - wav = wav.mean(-1) - assert wav.ndim == 1, wav.ndim - assert sr == self.task.cfg.sample_rate, sr - return wav - - def get_feats(self, file_path): - x = self.read_audio(file_path) - with torch.no_grad(): - source = torch.from_numpy(x).view(1, -1).float().cuda() - res = self.model( - source=source, mask=False, features_only=True, layer=self.layer - ) - return res["layer_results"][self.layer][0].squeeze(1) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/tools/README.md b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/tools/README.md deleted file mode 100644 index 385834841cc1376bef5bc73b2613806ed5cde38c..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/tools/README.md +++ /dev/null @@ -1,25 +0,0 @@ -# GSLM Tools - -## Resynthesis -You can use the command line tool below to input an audio file and get the resynthesized audio. This tool implements the unsupervised method for resynthesis described in the paper. The way to invoke the command line tool is shown below. -``` -FAIRSEQ_ROOT= -TYPE= -ACOUSTIC_MODEL_PATH= -LAYER= -KM_MODEL_PATH= -TTS_MODEL_PATH= -# A text file containing the codes, one per line -CODE_DICT_PATH= -WAVEGLOW_PATH= - -PYTHONPATH=${FAIRSEQ_ROOT}:${FAIRSEQ_ROOT}/examples/textless_nlp/gslm/unit2speech python ${FAIRSEQ_ROOT}/examples/textless_nlp/gslm/tools/resynthesize_speech.py \ - --feature_type $TYPE \ - --acoustic_model_path $ACOUSTIC_MODEL_PATH \ - --layer $LAYER \ - --kmeans_model_path $KM_MODEL_PATH \ - --tts_model_path $TTS_MODEL_PATH \ - --code_dict_path $CODE_DICT_PATH \ - --waveglow_path $WAVEGLOW_PATH \ - --max_decoder_steps 2000 -``` \ No newline at end of file diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/tools/resynthesize_speech.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/tools/resynthesize_speech.py deleted file mode 100644 index 309877212e19218fe400a4f12a95258ced914ed7..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/tools/resynthesize_speech.py +++ /dev/null @@ -1,132 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import gc -import logging -import os - -import joblib -import soundfile as sf -import torch -from examples.textless_nlp.gslm.speech2unit.pretrained.utils import get_feature_reader -from examples.textless_nlp.gslm.unit2speech.tts_data import TacotronInputDataset -from examples.textless_nlp.gslm.unit2speech.utils import ( - load_tacotron, - load_waveglow, - synthesize_audio, -) - - -def get_logger(): - log_format = "[%(asctime)s] [%(levelname)s]: %(message)s" - logging.basicConfig(format=log_format, level=logging.INFO) - logger = logging.getLogger(__name__) - return logger - - -def get_parser(): - parser = argparse.ArgumentParser(description="GSLM U2S tool") - parser.add_argument( - "--feature_type", - type=str, - choices=["logmel", "hubert", "w2v2", "cpc"], - default=None, - required=True, - help="Acoustic feature type", - ) - parser.add_argument( - "--acoustic_model_path", - type=str, - help="Pretrained acoustic model checkpoint", - ) - parser.add_argument("--layer", type=int, help="Layer of acoustic model") - parser.add_argument( - "--kmeans_model_path", - type=str, - required=True, - help="K-means model file path to use for inference", - ) - parser.add_argument( - "--tts_model_path", - type=str, - help="TTS model file path to use for inference", - ) - parser.add_argument( - "--code_dict_path", - type=str, - help="Code dict file path to use for inference", - ) - parser.add_argument( - "--waveglow_path", - type=str, - help="Waveglow (vocoder) model file path to use for inference", - ) - parser.add_argument("--max_decoder_steps", type=int, default=2000) - parser.add_argument("--denoiser_strength", type=float, default=0.1) - return parser - - -################################################ -def main(args, logger): - # Acoustic Model - logger.info(f"Loading acoustic model from {args.tts_model_path}...") - feature_reader_cls = get_feature_reader(args.feature_type) - reader = feature_reader_cls( - checkpoint_path=args.acoustic_model_path, layer=args.layer - ) - - # K-means Model - logger.info(f"Loading K-means model from {args.kmeans_model_path} ...") - kmeans_model = joblib.load(open(args.kmeans_model_path, "rb")) - kmeans_model.verbose = False - - # TTS Model - logger.info(f"Loading TTS model from {args.tts_model_path}...") - tacotron_model, sample_rate, hparams = load_tacotron( - tacotron_model_path=args.tts_model_path, - max_decoder_steps=args.max_decoder_steps, - ) - - # Waveglow Model - logger.info(f"Loading Waveglow model from {args.waveglow_path}...") - waveglow, denoiser = load_waveglow(waveglow_path=args.waveglow_path) - - # Dataset - if not os.path.exists(hparams.code_dict): - hparams.code_dict = args.code_dict_path - tts_dataset = TacotronInputDataset(hparams) - - iters = 0 - while True: - in_file_path = input("Input: Enter the full file path of audio file...\n") - out_file_path = input("Output: Enter the full file path of audio file...\n") - feats = reader.get_feats(in_file_path).cpu().numpy() - iters += 1 - if iters == 1000: - gc.collect() - torch.cuda.empty_cache() - - quantized_units = kmeans_model.predict(feats) - quantized_units_str = " ".join(map(str, quantized_units)) - - tts_input = tts_dataset.get_tensor(quantized_units_str) - mel, aud, aud_dn, has_eos = synthesize_audio( - tacotron_model, - waveglow, - denoiser, - tts_input.unsqueeze(0), - strength=args.denoiser_strength, - ) - sf.write(f"{out_file_path}", aud_dn[0].cpu().float().numpy(), sample_rate) - logger.info("Resynthesis done!\n") - - -if __name__ == "__main__": - parser = get_parser() - args = parser.parse_args() - logger = get_logger() - logger.info(args) - main(args, logger) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/ulm/README.md b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/ulm/README.md deleted file mode 100644 index 01459121cebefc61fdc2eae201462aa78d699111..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/ulm/README.md +++ /dev/null @@ -1,72 +0,0 @@ -# Unit Language Model (ULM) - -Here you can find links to the pre-trained ULMs and instructions on training new models using fairseq. At the end of the page, we also share how to run sampling for those models and provide pointers to the transcribed prompts we used. - -## Pre-trained models - -Using the links below, you can download pre-trained models for various unit types and vocabulary sizes: - -| | 50 | 100 | 200 -|-|-|-|- -| LogMel Filterbank | [download](https://dl.fbaipublicfiles.com/textless_nlp/gslm/logmel/lm_km50/logmel50_lm.tgz) | [download](https://dl.fbaipublicfiles.com/textless_nlp/gslm/logmel/lm_km100/logmel100_lm.tgz) | [download](https://dl.fbaipublicfiles.com/textless_nlp/gslm/logmel/lm_km200/logmel200_lm.tgz) -| Modified CPC | [download](https://dl.fbaipublicfiles.com/textless_nlp/gslm/cpc/lm_km50/cpc50_lm.tgz) | [download](https://dl.fbaipublicfiles.com/textless_nlp/gslm/cpc/lm_km100/cpc100_lm.tgz) | [download](https://dl.fbaipublicfiles.com/textless_nlp/gslm/cpc/lm_km200/cpc200_lm.tgz) -| HuBERT | [download](https://dl.fbaipublicfiles.com/textless_nlp/gslm/hubert/lm_km50/hubert50_lm.tgz) | [download](https://dl.fbaipublicfiles.com/textless_nlp/gslm/hubert/lm_km100/hubert100_lm.tgz) | [download](https://dl.fbaipublicfiles.com/textless_nlp/gslm/hubert/lm_km200/hubert200_lm.tgz) -| Wav2Vec 2.0 | [download](https://dl.fbaipublicfiles.com/textless_nlp/gslm/w2v2/lm_km50/w2v2_50_lm.tgz) | [download](https://dl.fbaipublicfiles.com/textless_nlp/gslm/w2v2/lm_km100/w2v2_100_lm.tgz) | [download](https://dl.fbaipublicfiles.com/textless_nlp/gslm/w2v2/lm_km200/w2v2_200_lm.tgz) - - -## Preprocessing data -Assuming that unit-transcribed train, valid, and test sets are located in `data/train.txt`, `data/valid.txt`, and `data/test.txt`, respectively, -we run the following command to get a preprocessed version of the datast in `data-bin`: - -```bash -fairseq-preprocess --only-source \ - --trainpref data/train.txt --validpref data/valid.txt --testpref data/test.txt \ - --destdir data-bin/ --workers 40 -``` -As a result, the `data-bin` directory should appear. - -## Fitting a Unit Language Model (ULM) -As an ULM, we train a standard fairseq Transformer LM. Assuming 8 GPUs used for training, a good starting point for an ULM training would be: -```bash - fairseq-train data-bin/ \ - --task=language_modeling \ - --arch=transformer_lm_big \ - --share-decoder-input-output-embed \ - --dropout=0.1 \ - --attention-dropout=0.1 \ - --optimizer=adam \ - --adam-betas='(0.9, 0.98)' \ - --clip-norm=1.0 \ - --lr=0.0005 \ - --lr-scheduler=inverse_sqrt \ - --warmup-updates=4000 \ - --warmup-init-lr=1e-07 \ - --tokens-per-sample=3072 \ - --update-freq=16 \ - --max-tokens=4096 \ - --num-workers=4 \ - --skip-invalid-size-inputs-valid-test \ - --max-update=500000 \ - --log-interval=10 \ - --seed=100501 \ - --fp16 \ - --sample-break-mode=eos -``` -This command will train a Transformer-large model (12 layers). You can train other standard LM models provided by fairseq, e.g. specify `--arch=transformer_lm` to train a smaller (6-layer) Transformer model. When training with a different number of GPUs, it might be a good idea to adjust the `update-freq` parameter. To save the GPU memory at an expense of additional computation, it can be useful to enable activation checkpointing with `--checkpoint-activations`. - -## Sampling from an ULM -Once an ULM was trained, we can use it for generating new utterances. Suppose, that the prompts are given in a file named `prompts.txt`. Then we can sample continuations by running the following command: - -```bash - python sample.py data-bin/ \ - --path=checkpoints/checkpoint_best.pt --task=language_modeling --sampling --temperature=0.7 \ - --seed=1 --prompts=prompts.txt --output=samples.txt --max-len-a=0 --max-len-b=500 \ - --prefix-size=-1 --batch-size=16 --fp16 --samples-per-prompt=10 -``` -Here, `--prefix-size` controls the number of tokens that are used to prime the ULM. When set to a positive value, the sampling script will take first `prefix-size` tokens to prompt the ULM; with `0` it runs unconditional sampling and with `-1` the entire prompt is used. -`--samples-per-prompt` specifies how many utterances are generated with every prompt which can be useful when generating multiple prompt continuations. In this command, `--max-len-a` and `--max-len-b` control the number of generated tokens. - -When using a pretrained model from above, `data-bin` should point to the unpacked directory (with `dict.txt` file). - -Evaluation-time, to generate prompts, we used utterances from LibriSpeech dev-clean and test-clean that are longer than 6s. We took first 3s from an utterance as a prompt. Unit transcripts of those prompts can be downloaded here: [[dev]](https://dl.fbaipublicfiles.com/textless_nlp/gslm/eval_data/dev_prompts.tgz) [[test]](https://dl.fbaipublicfiles.com/textless_nlp/gslm/eval_data/test_prompts.tgz) - diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/ulm/sample.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/ulm/sample.py deleted file mode 100644 index 9556c43c03ce6f2a06abd6f76f96321735cd8ce1..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/ulm/sample.py +++ /dev/null @@ -1,173 +0,0 @@ -#!/usr/bin/env python3 -u -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. -""" -Sample from a trained LM; hacked fairseq-interactive -""" -import ast -import os -from collections import namedtuple - -import numpy as np -import tqdm -from fairseq import checkpoint_utils, options, tasks, utils - -Batch = namedtuple('Batch', 'ids src_tokens src_lengths') -Translation = namedtuple('Translation', 'src_str hypos pos_scores alignments') - - -def make_batches(lines, args, task, max_positions): - tokens = [ - task.source_dictionary.encode_line( - src_str, add_if_not_exist=False - ).long() - for src_str in lines - ] - lengths = [t.numel() for t in tokens] - itr = task.get_batch_iterator( - dataset=task.build_dataset_for_inference(tokens, lengths), - max_tokens=args.dataset.max_tokens, - max_sentences=args.dataset.batch_size, - max_positions=max_positions, - ignore_invalid_inputs=args.dataset.skip_invalid_size_inputs_valid_test - ).next_epoch_itr(shuffle=False) - for batch in itr: - yield Batch( - ids=batch['id'], - src_tokens=batch['net_input']['src_tokens'], src_lengths=batch['net_input']['src_lengths'], - ) - - -def main(args): - arg_prompts = args.prompts - arg_output = args.output - arg_debug = args.debug - arg_sample_size = args.samples_per_prompt - - try: - from fairseq.dataclass.utils import convert_namespace_to_omegaconf - args = convert_namespace_to_omegaconf(args) - except: - pass - - # if args.max_tokens is None and args.max_sentences is None: - if args.common.seed is not None: - np.random.seed(args.common.seed) - utils.set_torch_seed(args.common.seed) - - if args.generation.sampling: - args.generation.nbest = args.generation.beam = arg_sample_size - - task = tasks.setup_task(args.task) - - overrides = ast.literal_eval(args.common_eval.model_overrides) - - models, _model_args = checkpoint_utils.load_model_ensemble( - args.common_eval.path.split(os.pathsep), - arg_overrides=overrides, - task=task, - suffix=getattr(args, "checkpoint_suffix", ""), - ) - - # Set dictionaries - src_dict = task.source_dictionary - tgt_dict = task.target_dictionary - - # Optimize ensemble for generation - for model in models: - model.prepare_for_inference_(args) - model.cuda() - - # Load alignment dictionary for unknown word replacement - # (None if no unknown word replacement, empty if no path to align dictionary) - align_dict = utils.load_align_dict(args.generation.replace_unk) - - max_positions = utils.resolve_max_positions( - task.max_positions(), - *[model.max_positions() for model in models] - ) - - output_file = open(arg_output, 'w') - - with open(arg_prompts, 'r') as fin: - lines = fin.readlines() - - split = [x.split('|', 1) for x in lines] - seq_id = [x[0] for x in split] - prompts = [x[1] for x in split] - - if args.generation.prefix_size >= 0: - prompts = [' '.join(l.split()[:args.generation.prefix_size]) - for l in prompts] - - if arg_debug: - prompts = prompts[:10] - - generator = task.build_generator(models, args.generation) - - start_id = 0 - pbar = tqdm.tqdm(total=len(prompts)) - for batch in make_batches(prompts, args, task, max_positions): - src_tokens = batch.src_tokens - src_lengths = batch.src_lengths - src_tokens = src_tokens.cuda() - src_lengths = src_lengths.cuda() - - sample = { - 'net_input': { - 'src_tokens': src_tokens, - 'src_lengths': src_lengths, - }, - } - - results = [] - translations = task.inference_step(generator, models, sample) - for i, (id, hypos) in enumerate(zip(batch.ids.tolist(), translations)): - src_tokens_i = utils.strip_pad(src_tokens[i], tgt_dict.pad()) - results.append((i + start_id, src_tokens_i, hypos)) - - # sort output to match input order - for id, src_tokens, hypos in sorted(results, key=lambda x: x[0]): - if src_dict is not None: - src_str = src_dict.string( - src_tokens, args.common_eval.post_process) - - # Process top predictions - for hypo_id, hypo in enumerate(hypos): - _hypo_tokens, hypo_str, _alignment = utils.post_process_prediction( - hypo_tokens=hypo['tokens'].int().cpu(), - src_str=src_str, - alignment=hypo['alignment'], - align_dict=align_dict, - tgt_dict=tgt_dict, - remove_bpe=args.common_eval.post_process, - ) - - detok_hypo_str = hypo_str - utterance = detok_hypo_str - print(f'{seq_id[id]}__{hypo_id}|{utterance}', file=output_file) - pbar.update(1) - start_id += len(results) - - # output_file.close() - - -def cli_main(): - parser = options.get_interactive_generation_parser() - parser.add_argument('--prompts', type=str, default=None, required=True) - parser.add_argument('--output', type=str, default=None, required=True) - parser.add_argument('--debug', action='store_true') - parser.add_argument('--samples-per-prompt', type=int, default=1) - - args = options.parse_args_and_arch(parser) - - np.random.seed(args.seed) - utils.set_torch_seed(args.seed) - - main(args) - - -if __name__ == '__main__': - cli_main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/README.md b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/README.md deleted file mode 100644 index e61601392bd77dc747e43182b408bb941bc4728b..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/README.md +++ /dev/null @@ -1,40 +0,0 @@ -# Unit to Speech Model (unit2speech) - -Unit to speech model is modified Tacotron2 model that learns to synthesize speech from discrete speech units. All models are trained on quantized [LJSpeech](https://keithito.com/LJ-Speech-Dataset/). - -Upstream Units | Download Links | model md5 -|-|-|- -Log Mel Filterbank + KM50 | [model](https://dl.fbaipublicfiles.com/textless_nlp/gslm/logmel/tts_km50/tts_checkpoint_best.pt) - [code_dict](https://dl.fbaipublicfiles.com/textless_nlp/gslm/logmel/tts_km50/code_dict) | 932b3b8527c0125f5f964b57762eba49 -Log Mel Filterbank + KM100 | [model](https://dl.fbaipublicfiles.com/textless_nlp/gslm/logmel/tts_km100/tts_checkpoint_best.pt) - [code_dict](https://dl.fbaipublicfiles.com/textless_nlp/gslm/logmel/tts_km100/code_dict) | cde0b0d278a39011d0acbd5df27abdf4 -Log Mel Filterbank + KM200 | [model](https://dl.fbaipublicfiles.com/textless_nlp/gslm/logmel/tts_km200/tts_checkpoint_best.pt) - [code_dict](https://dl.fbaipublicfiles.com/textless_nlp/gslm/logmel/tts_km200/code_dict) | dba0f1d4de64bc7976718834010b23e7 -Modified CPC + KM50 | [model](https://dl.fbaipublicfiles.com/textless_nlp/gslm/cpc/tts_km50/tts_checkpoint_best.pt) - [code_dict](https://dl.fbaipublicfiles.com/textless_nlp/gslm/cpc/tts_km50/code_dict) | a585e8dd8890ea56164f17635dd8e613 -Modified CPC + KM100 | [model](https://dl.fbaipublicfiles.com/textless_nlp/gslm/cpc/tts_km100/tts_checkpoint_best.pt) - [code_dict](https://dl.fbaipublicfiles.com/textless_nlp/gslm/cpc/tts_km100/code_dict) | 5c0ee2869b4f483d17f37f1a41a548e0 -Modified CPC + KM200 | [model](https://dl.fbaipublicfiles.com/textless_nlp/gslm/cpc/tts_km200/tts_checkpoint_best.pt) - [code_dict](https://dl.fbaipublicfiles.com/textless_nlp/gslm/cpc/tts_km200/code_dict) | 2f0c9951cf37020d9464514bff48bc5d -HuBERT Base + KM50 | [model](https://dl.fbaipublicfiles.com/textless_nlp/gslm/hubert/tts_km50/tts_checkpoint_best.pt) - [code_dict](https://dl.fbaipublicfiles.com/textless_nlp/gslm/hubert/tts_km50/code_dict) | 85ffce8baec5aa90035ab696fe676fce -HuBERT Base + KM100 | [model](https://dl.fbaipublicfiles.com/textless_nlp/gslm/hubert/tts_km100/tts_checkpoint_best.pt) - [code_dict](https://dl.fbaipublicfiles.com/textless_nlp/gslm/hubert/tts_km100/code_dict) | df4a9c6ffd1bb00c91405432c234aba3 -HuBERT Base + KM200 | [model](https://dl.fbaipublicfiles.com/textless_nlp/gslm/hubert/tts_km200/tts_checkpoint_best.pt) - [code_dict](https://dl.fbaipublicfiles.com/textless_nlp/gslm/hubert/tts_km200/code_dict) | ac72f2c0c563589819bec116c7f8d274 -wav2vec 2.0 Large + KM50 | [model](https://dl.fbaipublicfiles.com/textless_nlp/gslm/w2v2/tts_km50/tts_checkpoint_best.pt) - [code_dict](https://dl.fbaipublicfiles.com/textless_nlp/gslm/w2v2/tts_km50/code_dict) | e3503d0ad822b2c24b89f68b857fedff -wav2vec 2.0 Large + KM100 | [model](https://dl.fbaipublicfiles.com/textless_nlp/gslm/w2v2/tts_km100/tts_checkpoint_best.pt) - [code_dict](https://dl.fbaipublicfiles.com/textless_nlp/gslm/w2v2/tts_km100/code_dict) | eb3666e456ae4c96bf2a1eec825c13ed -wav2vec 2.0 Large + KM200 | [model](https://dl.fbaipublicfiles.com/textless_nlp/gslm/w2v2/tts_km200/tts_checkpoint_best.pt) - [code_dict](https://dl.fbaipublicfiles.com/textless_nlp/gslm/w2v2/tts_km200/code_dict) | 777d343e963c4d64f04d78eef032f4e8 - -## Run inference using a unit2speech model -* Install librosa, unidecode and inflect using `pip install librosa, unidecode, inflect` -* Download [Waveglow checkpoint](https://dl.fbaipublicfiles.com/textless_nlp/gslm/waveglow_256channels_new.pt). This is the vocoder. - -Sample commnd to run inference using trained unit2speech models. Please note that the quantized audio to synthesized should be using the same units as the unit2speech model was trained with. -``` -FAIRSEQ_ROOT= -TTS_MODEL_PATH= -QUANTIZED_UNIT_PATH= -OUT_DIR= -WAVEGLOW_PATH= -CODE_DICT_PATH= - -PYTHONPATH=${FAIRSEQ_ROOT}:${FAIRSEQ_ROOT}/examples/textless_nlp/gslm/unit2speech python ${FAIRSEQ_ROOT}/examples/textless_nlp/gslm/unit2speech/synthesize_audio_from_units.py \ - --tts_model_path $TTS_MODEL_PATH \ - --quantized_unit_path $QUANTIZED_UNIT_PATH \ - --out_audio_dir $OUT_DIR \ - --waveglow_path $WAVEGLOW_PATH \ - --code_dict_path $CODE_DICT_PATH \ - --max_decoder_steps 2000 -``` diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/convert_to_16k.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/convert_to_16k.py deleted file mode 100644 index 1dea97f402f1e4dbfa9a2a37b18c0a4c18fbb0cb..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/convert_to_16k.py +++ /dev/null @@ -1,58 +0,0 @@ -import os -import shlex -import subprocess -from pathlib import Path -from time import time - -import progressbar - - -def find_all_files(path_dir, extension): - out = [] - for root, dirs, filenames in os.walk(path_dir): - for f in filenames: - if f.endswith(extension): - out.append(((str(Path(f).stem)), os.path.join(root, f))) - return out - -def convert16k(inputfile, outputfile16k): - command = ('sox -c 1 -b 16 {} -t wav {} rate 16k'.format(inputfile, outputfile16k)) - subprocess.call(shlex.split(command)) - -if __name__ == "__main__": - import argparse - - parser = argparse.ArgumentParser(description='Convert to wav 16k audio using sox.') - parser.add_argument('input_dir', type=str, - help='Path to the input dir.') - parser.add_argument('output_dir', type=str, - help='Path to the output dir.') - parser.add_argument('--extension', type=str, default='wav', - help='Audio file extension in the input. Default: mp3') - args = parser.parse_args() - - # Find all sequences - print(f"Finding all audio files with extension '{args.extension}' from {args.input_dir}...") - audio_files = find_all_files(args.input_dir, args.extension) - print(f"Done! Found {len(audio_files)} files.") - - # Convert to relative path - audio_files = [os.path.relpath(file[-1], start=args.input_dir) for file in audio_files] - - # Create all the directories needed - rel_dirs_set = set([os.path.dirname(file) for file in audio_files]) - for rel_dir in rel_dirs_set: - Path(os.path.join(args.output_dir, rel_dir)).mkdir(parents=True, exist_ok=True) - - # Converting wavs files - print("Converting the audio to wav files...") - bar = progressbar.ProgressBar(maxval=len(audio_files)) - bar.start() - start_time = time() - for index, file in enumerate(audio_files): - bar.update(index) - input_file = os.path.join(args.input_dir, file) - output_file = os.path.join(args.output_dir, os.path.splitext(file)[0]+".wav") - convert16k(input_file, output_file) - bar.finish() - print(f"...done {len(audio_files)} files in {time()-start_time} seconds.") \ No newline at end of file diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/glow.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/glow.py deleted file mode 100644 index 04504037a4e4a92d5a8c088ffa20f093abe32300..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/glow.py +++ /dev/null @@ -1,312 +0,0 @@ -# ***************************************************************************** -# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of the NVIDIA CORPORATION nor the -# names of its contributors may be used to endorse or promote products -# derived from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY -# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# ***************************************************************************** -import copy - -import torch -import torch.nn.functional as F -from torch.autograd import Variable - - -@torch.jit.script -def fused_add_tanh_sigmoid_multiply(input_a, input_b, n_channels): - n_channels_int = n_channels[0] - in_act = input_a+input_b - t_act = torch.tanh(in_act[:, :n_channels_int, :]) - s_act = torch.sigmoid(in_act[:, n_channels_int:, :]) - acts = t_act * s_act - return acts - - -class WaveGlowLoss(torch.nn.Module): - def __init__(self, sigma=1.0): - super(WaveGlowLoss, self).__init__() - self.sigma = sigma - - def forward(self, model_output): - z, log_s_list, log_det_W_list = model_output - for i, log_s in enumerate(log_s_list): - if i == 0: - log_s_total = torch.sum(log_s) - log_det_W_total = log_det_W_list[i] - else: - log_s_total = log_s_total + torch.sum(log_s) - log_det_W_total += log_det_W_list[i] - - loss = torch.sum(z*z)/(2*self.sigma*self.sigma) - log_s_total - log_det_W_total - return loss/(z.size(0)*z.size(1)*z.size(2)) - - -class Invertible1x1Conv(torch.nn.Module): - """ - The layer outputs both the convolution, and the log determinant - of its weight matrix. If reverse=True it does convolution with - inverse - """ - def __init__(self, c): - super(Invertible1x1Conv, self).__init__() - self.conv = torch.nn.Conv1d(c, c, kernel_size=1, stride=1, padding=0, - bias=False) - - # Sample a random orthonormal matrix to initialize weights - W = torch.qr(torch.FloatTensor(c, c).normal_())[0] - - # Ensure determinant is 1.0 not -1.0 - if torch.det(W) < 0: - W[:,0] = -1*W[:,0] - W = W.view(c, c, 1) - self.conv.weight.data = W - - def forward(self, z, reverse=False): - # shape - batch_size, group_size, n_of_groups = z.size() - - W = self.conv.weight.squeeze() - - if reverse: - if not hasattr(self, 'W_inverse'): - # Reverse computation - W_inverse = W.float().inverse() - W_inverse = Variable(W_inverse[..., None]) - if z.type() == 'torch.cuda.HalfTensor': - W_inverse = W_inverse.half() - self.W_inverse = W_inverse - z = F.conv1d(z, self.W_inverse, bias=None, stride=1, padding=0) - return z - else: - # Forward computation - log_det_W = batch_size * n_of_groups * torch.logdet(W) - z = self.conv(z) - return z, log_det_W - - -class WN(torch.nn.Module): - """ - This is the WaveNet like layer for the affine coupling. The primary difference - from WaveNet is the convolutions need not be causal. There is also no dilation - size reset. The dilation only doubles on each layer - """ - def __init__(self, n_in_channels, n_mel_channels, n_layers, n_channels, - kernel_size): - super(WN, self).__init__() - assert(kernel_size % 2 == 1) - assert(n_channels % 2 == 0) - self.n_layers = n_layers - self.n_channels = n_channels - self.in_layers = torch.nn.ModuleList() - self.res_skip_layers = torch.nn.ModuleList() - - start = torch.nn.Conv1d(n_in_channels, n_channels, 1) - start = torch.nn.utils.weight_norm(start, name='weight') - self.start = start - - # Initializing last layer to 0 makes the affine coupling layers - # do nothing at first. This helps with training stability - end = torch.nn.Conv1d(n_channels, 2*n_in_channels, 1) - end.weight.data.zero_() - end.bias.data.zero_() - self.end = end - - cond_layer = torch.nn.Conv1d(n_mel_channels, 2*n_channels*n_layers, 1) - self.cond_layer = torch.nn.utils.weight_norm(cond_layer, name='weight') - - for i in range(n_layers): - dilation = 2 ** i - padding = int((kernel_size*dilation - dilation)/2) - in_layer = torch.nn.Conv1d(n_channels, 2*n_channels, kernel_size, - dilation=dilation, padding=padding) - in_layer = torch.nn.utils.weight_norm(in_layer, name='weight') - self.in_layers.append(in_layer) - - - # last one is not necessary - if i < n_layers - 1: - res_skip_channels = 2*n_channels - else: - res_skip_channels = n_channels - res_skip_layer = torch.nn.Conv1d(n_channels, res_skip_channels, 1) - res_skip_layer = torch.nn.utils.weight_norm(res_skip_layer, name='weight') - self.res_skip_layers.append(res_skip_layer) - - def forward(self, forward_input): - audio, spect = forward_input - audio = self.start(audio) - output = torch.zeros_like(audio) - n_channels_tensor = torch.IntTensor([self.n_channels]) - - spect = self.cond_layer(spect) - - for i in range(self.n_layers): - spect_offset = i*2*self.n_channels - acts = fused_add_tanh_sigmoid_multiply( - self.in_layers[i](audio), - spect[:,spect_offset:spect_offset+2*self.n_channels,:], - n_channels_tensor) - - res_skip_acts = self.res_skip_layers[i](acts) - if i < self.n_layers - 1: - audio = audio + res_skip_acts[:,:self.n_channels,:] - output = output + res_skip_acts[:,self.n_channels:,:] - else: - output = output + res_skip_acts - - return self.end(output) - - -class WaveGlow(torch.nn.Module): - def __init__(self, n_mel_channels, n_flows, n_group, n_early_every, - n_early_size, WN_config): - super(WaveGlow, self).__init__() - - self.upsample = torch.nn.ConvTranspose1d(n_mel_channels, - n_mel_channels, - 1024, stride=256) - assert(n_group % 2 == 0) - self.n_flows = n_flows - self.n_group = n_group - self.n_early_every = n_early_every - self.n_early_size = n_early_size - self.WN = torch.nn.ModuleList() - self.convinv = torch.nn.ModuleList() - - n_half = int(n_group/2) - - # Set up layers with the right sizes based on how many dimensions - # have been output already - n_remaining_channels = n_group - for k in range(n_flows): - if k % self.n_early_every == 0 and k > 0: - n_half = n_half - int(self.n_early_size/2) - n_remaining_channels = n_remaining_channels - self.n_early_size - self.convinv.append(Invertible1x1Conv(n_remaining_channels)) - self.WN.append(WN(n_half, n_mel_channels*n_group, **WN_config)) - self.n_remaining_channels = n_remaining_channels # Useful during inference - - def forward(self, forward_input): - """ - forward_input[0] = mel_spectrogram: batch x n_mel_channels x frames - forward_input[1] = audio: batch x time - """ - spect, audio = forward_input - - # Upsample spectrogram to size of audio - spect = self.upsample(spect) - assert(spect.size(2) >= audio.size(1)) - if spect.size(2) > audio.size(1): - spect = spect[:, :, :audio.size(1)] - - spect = spect.unfold(2, self.n_group, self.n_group).permute(0, 2, 1, 3) - spect = spect.contiguous().view(spect.size(0), spect.size(1), -1).permute(0, 2, 1) - - audio = audio.unfold(1, self.n_group, self.n_group).permute(0, 2, 1) - output_audio = [] - log_s_list = [] - log_det_W_list = [] - - for k in range(self.n_flows): - if k % self.n_early_every == 0 and k > 0: - output_audio.append(audio[:,:self.n_early_size,:]) - audio = audio[:,self.n_early_size:,:] - - audio, log_det_W = self.convinv[k](audio) - log_det_W_list.append(log_det_W) - - n_half = int(audio.size(1)/2) - audio_0 = audio[:,:n_half,:] - audio_1 = audio[:,n_half:,:] - - output = self.WN[k]((audio_0, spect)) - log_s = output[:, n_half:, :] - b = output[:, :n_half, :] - audio_1 = torch.exp(log_s)*audio_1 + b - log_s_list.append(log_s) - - audio = torch.cat([audio_0, audio_1],1) - - output_audio.append(audio) - return torch.cat(output_audio,1), log_s_list, log_det_W_list - - def infer(self, spect, sigma=1.0): - spect = self.upsample(spect) - # trim conv artifacts. maybe pad spec to kernel multiple - time_cutoff = self.upsample.kernel_size[0] - self.upsample.stride[0] - spect = spect[:, :, :-time_cutoff] - - spect = spect.unfold(2, self.n_group, self.n_group).permute(0, 2, 1, 3) - spect = spect.contiguous().view(spect.size(0), spect.size(1), -1).permute(0, 2, 1) - - if spect.type() == 'torch.cuda.HalfTensor': - audio = torch.cuda.HalfTensor(spect.size(0), - self.n_remaining_channels, - spect.size(2)).normal_() - else: - audio = torch.cuda.FloatTensor(spect.size(0), - self.n_remaining_channels, - spect.size(2)).normal_() - - audio = torch.autograd.Variable(sigma*audio) - - for k in reversed(range(self.n_flows)): - n_half = int(audio.size(1)/2) - audio_0 = audio[:,:n_half,:] - audio_1 = audio[:,n_half:,:] - - output = self.WN[k]((audio_0, spect)) - - s = output[:, n_half:, :] - b = output[:, :n_half, :] - audio_1 = (audio_1 - b)/torch.exp(s) - audio = torch.cat([audio_0, audio_1],1) - - audio = self.convinv[k](audio, reverse=True) - - if k % self.n_early_every == 0 and k > 0: - if spect.type() == 'torch.cuda.HalfTensor': - z = torch.cuda.HalfTensor(spect.size(0), self.n_early_size, spect.size(2)).normal_() - else: - z = torch.cuda.FloatTensor(spect.size(0), self.n_early_size, spect.size(2)).normal_() - audio = torch.cat((sigma*z, audio),1) - - audio = audio.permute(0,2,1).contiguous().view(audio.size(0), -1).data - return audio - - @staticmethod - def remove_weightnorm(model): - waveglow = model - for WN in waveglow.WN: - WN.start = torch.nn.utils.remove_weight_norm(WN.start) - WN.in_layers = remove(WN.in_layers) - WN.cond_layer = torch.nn.utils.remove_weight_norm(WN.cond_layer) - WN.res_skip_layers = remove(WN.res_skip_layers) - return waveglow - - -def remove(conv_list): - new_conv_list = torch.nn.ModuleList() - for old_conv in conv_list: - old_conv = torch.nn.utils.remove_weight_norm(old_conv) - new_conv_list.append(old_conv) - return new_conv_list diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/multiproc.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/multiproc.py deleted file mode 100644 index c034814c8777d78219dc15f66073eec04537c173..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/multiproc.py +++ /dev/null @@ -1,28 +0,0 @@ -import os -import subprocess -import sys -import time - -import torch - -argslist = list(sys.argv)[1:] -log_dir = argslist[-1] -num_gpus = torch.cuda.device_count() -argslist.append('--n_gpus={}'.format(num_gpus)) -workers = [] -job_id = time.strftime("%Y_%m_%d-%H%M%S") -argslist.append("--group_name=group_{}".format(job_id)) - -print("GPU log directory is {}".format(log_dir)) -os.makedirs(log_dir, exist_ok=True) -for i in range(num_gpus): - argslist.append('--rank={}'.format(i)) - stdout = None if i == 0 else open("{}/{}_GPU_{}.log".format(log_dir, job_id, i), - "w") - print(argslist) - p = subprocess.Popen([str(sys.executable)]+argslist, stdout=stdout) - workers.append(p) - argslist = argslist[:-1] - -for p in workers: - p.wait() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/synthesize_audio_from_units.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/synthesize_audio_from_units.py deleted file mode 100644 index 80730843bf6ae5bbb10e190df38941f39b14cb9a..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/synthesize_audio_from_units.py +++ /dev/null @@ -1,105 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import logging -import os - -import soundfile as sf -from examples.textless_nlp.gslm.unit2speech.tts_data import ( - TacotronInputDataset, -) -from examples.textless_nlp.gslm.unit2speech.utils import ( - load_quantized_audio_from_file, - load_tacotron, - load_waveglow, - synthesize_audio, -) - - -def get_logger(): - log_format = "[%(asctime)s] [%(levelname)s]: %(message)s" - logging.basicConfig(format=log_format, level=logging.INFO) - logger = logging.getLogger(__name__) - return logger - - -def get_parser(): - parser = argparse.ArgumentParser( - description="Wav2Vec 2.0 speech generator." - ) - parser.add_argument( - "--quantized_unit_path", - type=str, - help="K-means model file path to use for inference", - ) - parser.add_argument( - "--tts_model_path", - type=str, - help="TTS model file path to use for inference", - ) - parser.add_argument( - "--waveglow_path", - type=str, - help="Path to the waveglow checkpoint (vocoder).", - ) - parser.add_argument( - "--code_dict_path", - type=str, - help="Code dict file path to use for inference", - ) - parser.add_argument("--max_decoder_steps", type=int, default=2000) - parser.add_argument("--denoiser_strength", type=float, default=0.1) - parser.add_argument( - "--out_audio_dir", - type=str, - help="Output directory to dump audio files", - ) - - return parser - - -def main(args, logger): - # Load quantized audio - logger.info(f"Loading quantized audio from {args.quantized_unit_path}...") - names_batch, quantized_units_batch = load_quantized_audio_from_file( - file_path=args.quantized_unit_path - ) - - logger.info(f"Loading TTS model from {args.tts_model_path}...") - tacotron_model, sample_rate, hparams = load_tacotron( - tacotron_model_path=args.tts_model_path, - max_decoder_steps=args.max_decoder_steps, - ) - - logger.info(f"Loading Waveglow model from {args.waveglow_path}...") - waveglow, denoiser = load_waveglow(waveglow_path=args.waveglow_path) - - if not os.path.exists(hparams.code_dict): - hparams.code_dict = args.code_dict_path - tts_dataset = TacotronInputDataset(hparams) - - for name, quantized_units in zip(names_batch, quantized_units_batch): - quantized_units_str = " ".join(map(str, quantized_units)) - tts_input = tts_dataset.get_tensor(quantized_units_str) - mel, aud, aud_dn, has_eos = synthesize_audio( - tacotron_model, - waveglow, - denoiser, - tts_input.unsqueeze(0), - strength=args.denoiser_strength, - ) - out_file_path = os.path.join(args.out_audio_dir, f"{name}.wav") - sf.write( - f"{out_file_path}", aud_dn[0].cpu().float().numpy(), sample_rate - ) - - -if __name__ == "__main__": - parser = get_parser() - args = parser.parse_args() - logger = get_logger() - logger.info(args) - main(args, logger) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/tacotron2/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/tacotron2/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/tacotron2/audio_processing.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/tacotron2/audio_processing.py deleted file mode 100644 index 0613ca0874837dfd525abfecbd5abc6080a2c47a..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/tacotron2/audio_processing.py +++ /dev/null @@ -1,93 +0,0 @@ -import librosa.util as librosa_util -import numpy as np -import torch -from scipy.signal import get_window - - -def window_sumsquare(window, n_frames, hop_length=200, win_length=800, - n_fft=800, dtype=np.float32, norm=None): - """ - # from librosa 0.6 - Compute the sum-square envelope of a window function at a given hop length. - - This is used to estimate modulation effects induced by windowing - observations in short-time fourier transforms. - - Parameters - ---------- - window : string, tuple, number, callable, or list-like - Window specification, as in `get_window` - - n_frames : int > 0 - The number of analysis frames - - hop_length : int > 0 - The number of samples to advance between frames - - win_length : [optional] - The length of the window function. By default, this matches `n_fft`. - - n_fft : int > 0 - The length of each analysis frame. - - dtype : np.dtype - The data type of the output - - Returns - ------- - wss : np.ndarray, shape=`(n_fft + hop_length * (n_frames - 1))` - The sum-squared envelope of the window function - """ - if win_length is None: - win_length = n_fft - - n = n_fft + hop_length * (n_frames - 1) - x = np.zeros(n, dtype=dtype) - - # Compute the squared window at the desired length - win_sq = get_window(window, win_length, fftbins=True) - win_sq = librosa_util.normalize(win_sq, norm=norm)**2 - win_sq = librosa_util.pad_center(win_sq, n_fft) - - # Fill the envelope - for i in range(n_frames): - sample = i * hop_length - x[sample:min(n, sample + n_fft)] += win_sq[:max(0, min(n_fft, n - sample))] - return x - - -def griffin_lim(magnitudes, stft_fn, n_iters=30): - """ - PARAMS - ------ - magnitudes: spectrogram magnitudes - stft_fn: STFT class with transform (STFT) and inverse (ISTFT) methods - """ - - angles = np.angle(np.exp(2j * np.pi * np.random.rand(*magnitudes.size()))) - angles = angles.astype(np.float32) - angles = torch.autograd.Variable(torch.from_numpy(angles)) - signal = stft_fn.inverse(magnitudes, angles).squeeze(1) - - for i in range(n_iters): - _, angles = stft_fn.transform(signal) - signal = stft_fn.inverse(magnitudes, angles).squeeze(1) - return signal - - -def dynamic_range_compression(x, C=1, clip_val=1e-5): - """ - PARAMS - ------ - C: compression factor - """ - return torch.log(torch.clamp(x, min=clip_val) * C) - - -def dynamic_range_decompression(x, C=1): - """ - PARAMS - ------ - C: compression factor used to compress - """ - return torch.exp(x) / C diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/tacotron2/cleaners.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/tacotron2/cleaners.py deleted file mode 100644 index 10470d62ea0cc9406b293d31245340aebd9a03d5..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/tacotron2/cleaners.py +++ /dev/null @@ -1,91 +0,0 @@ -""" from https://github.com/keithito/tacotron """ - -''' -Cleaners are transformations that run over the input text at both training and eval time. - -Cleaners can be selected by passing a comma-delimited list of cleaner names as the "cleaners" -hyperparameter. Some cleaners are English-specific. You'll typically want to use: - 1. "english_cleaners" for English text - 2. "transliteration_cleaners" for non-English text that can be transliterated to ASCII using - the Unidecode library (https://pypi.python.org/pypi/Unidecode) - 3. "basic_cleaners" if you do not want to transliterate (in this case, you should also update - the symbols in symbols.py to match your data). -''' - -import re - -from unidecode import unidecode - -from .numbers import normalize_numbers - -# Regular expression matching whitespace: -_whitespace_re = re.compile(r'\s+') - -# List of (regular expression, replacement) pairs for abbreviations: -_abbreviations = [(re.compile('\\b%s\\.' % x[0], re.IGNORECASE), x[1]) for x in [ - ('mrs', 'misess'), - ('mr', 'mister'), - ('dr', 'doctor'), - ('st', 'saint'), - ('co', 'company'), - ('jr', 'junior'), - ('maj', 'major'), - ('gen', 'general'), - ('drs', 'doctors'), - ('rev', 'reverend'), - ('lt', 'lieutenant'), - ('hon', 'honorable'), - ('sgt', 'sergeant'), - ('capt', 'captain'), - ('esq', 'esquire'), - ('ltd', 'limited'), - ('col', 'colonel'), - ('ft', 'fort'), -]] - - -def expand_abbreviations(text): - for regex, replacement in _abbreviations: - text = re.sub(regex, replacement, text) - return text - - -def expand_numbers(text): - return normalize_numbers(text) - - -def lowercase(text): - return text.lower() - - -def collapse_whitespace(text): - return re.sub(_whitespace_re, ' ', text) - - -def convert_to_ascii(text): - return unidecode(text) - - -def basic_cleaners(text): - '''Basic pipeline that lowercases and collapses whitespace without transliteration.''' - text = lowercase(text) - text = collapse_whitespace(text) - return text - - -def transliteration_cleaners(text): - '''Pipeline for non-English text that transliterates to ASCII.''' - text = convert_to_ascii(text) - text = lowercase(text) - text = collapse_whitespace(text) - return text - - -def english_cleaners(text): - '''Pipeline for English text, including number and abbreviation expansion.''' - text = convert_to_ascii(text) - text = lowercase(text) - text = expand_numbers(text) - text = expand_abbreviations(text) - text = collapse_whitespace(text) - return text diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/tacotron2/cmudict.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/tacotron2/cmudict.py deleted file mode 100644 index 50a8448432d6b36f5345c27b285826dc69252009..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/tacotron2/cmudict.py +++ /dev/null @@ -1,64 +0,0 @@ -""" from https://github.com/keithito/tacotron """ - -import re - -valid_symbols = [ - 'AA', 'AA0', 'AA1', 'AA2', 'AE', 'AE0', 'AE1', 'AE2', 'AH', 'AH0', 'AH1', 'AH2', - 'AO', 'AO0', 'AO1', 'AO2', 'AW', 'AW0', 'AW1', 'AW2', 'AY', 'AY0', 'AY1', 'AY2', - 'B', 'CH', 'D', 'DH', 'EH', 'EH0', 'EH1', 'EH2', 'ER', 'ER0', 'ER1', 'ER2', 'EY', - 'EY0', 'EY1', 'EY2', 'F', 'G', 'HH', 'IH', 'IH0', 'IH1', 'IH2', 'IY', 'IY0', 'IY1', - 'IY2', 'JH', 'K', 'L', 'M', 'N', 'NG', 'OW', 'OW0', 'OW1', 'OW2', 'OY', 'OY0', - 'OY1', 'OY2', 'P', 'R', 'S', 'SH', 'T', 'TH', 'UH', 'UH0', 'UH1', 'UH2', 'UW', - 'UW0', 'UW1', 'UW2', 'V', 'W', 'Y', 'Z', 'ZH' -] - -_valid_symbol_set = set(valid_symbols) - - -class CMUDict: - '''Thin wrapper around CMUDict data. http://www.speech.cs.cmu.edu/cgi-bin/cmudict''' - def __init__(self, file_or_path, keep_ambiguous=True): - if isinstance(file_or_path, str): - with open(file_or_path, encoding='latin-1') as f: - entries = _parse_cmudict(f) - else: - entries = _parse_cmudict(file_or_path) - if not keep_ambiguous: - entries = {word: pron for word, pron in entries.items() if len(pron) == 1} - self._entries = entries - - - def __len__(self): - return len(self._entries) - - - def lookup(self, word): - '''Returns list of ARPAbet pronunciations of the given word.''' - return self._entries.get(word.upper()) - - - -_alt_re = re.compile(r'\([0-9]+\)') - - -def _parse_cmudict(file): - cmudict = {} - for line in file: - if len(line) and (line[0] >= 'A' and line[0] <= 'Z' or line[0] == "'"): - parts = line.split(' ') - word = re.sub(_alt_re, '', parts[0]) - pronunciation = _get_pronunciation(parts[1]) - if pronunciation: - if word in cmudict: - cmudict[word].append(pronunciation) - else: - cmudict[word] = [pronunciation] - return cmudict - - -def _get_pronunciation(s): - parts = s.strip().split(' ') - for part in parts: - if part not in _valid_symbol_set: - return None - return ' '.join(parts) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/tacotron2/layers.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/tacotron2/layers.py deleted file mode 100644 index 6d0ec3f6cda842690e61bd8f4ca53a519d400435..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/tacotron2/layers.py +++ /dev/null @@ -1,103 +0,0 @@ -import torch -from librosa.filters import mel as librosa_mel_fn - -from .audio_processing import dynamic_range_compression, dynamic_range_decompression -from .stft import STFT -from .utils import get_mask_from_lengths - - -class LinearNorm(torch.nn.Module): - def __init__(self, in_dim, out_dim, bias=True, w_init_gain='linear'): - super(LinearNorm, self).__init__() - self.linear_layer = torch.nn.Linear(in_dim, out_dim, bias=bias) - - torch.nn.init.xavier_uniform_( - self.linear_layer.weight, - gain=torch.nn.init.calculate_gain(w_init_gain)) - - def forward(self, x): - return self.linear_layer(x) - - -class ConvNorm(torch.nn.Module): - def __init__(self, in_channels, out_channels, kernel_size=1, stride=1, - padding=None, dilation=1, bias=True, w_init_gain='linear'): - super(ConvNorm, self).__init__() - if padding is None: - assert(kernel_size % 2 == 1) - padding = int(dilation * (kernel_size - 1) / 2) - - self.conv = torch.nn.Conv1d(in_channels, out_channels, - kernel_size=kernel_size, stride=stride, - padding=padding, dilation=dilation, - bias=bias) - - torch.nn.init.xavier_uniform_( - self.conv.weight, gain=torch.nn.init.calculate_gain(w_init_gain)) - - def forward(self, signal): - conv_signal = self.conv(signal) - return conv_signal - - -class GlobalAvgPool(torch.nn.Module): - def __init__(self): - super(GlobalAvgPool, self).__init__() - - def forward(self, x, lengths=None): - """Average pooling across time steps (dim=1) with optionally lengths. - Args: - x: torch.Tensor of shape (N, T, ...) - lengths: None or torch.Tensor of shape (N,) - dim: dimension to pool - """ - if lengths is None: - return x.mean(dim=1, keepdim=False) - else: - mask = get_mask_from_lengths(lengths).type(x.type()).to(x.device) - mask_shape = list(mask.size()) + [1 for _ in range(x.ndimension()-2)] - mask = mask.reshape(*mask_shape) - numer = (x * mask).sum(dim=1, keepdim=False) - denom = mask.sum(dim=1, keepdim=False) - return numer / denom - - -class TacotronSTFT(torch.nn.Module): - def __init__(self, filter_length=1024, hop_length=256, win_length=1024, - n_mel_channels=80, sampling_rate=22050, mel_fmin=0.0, - mel_fmax=8000.0): - super(TacotronSTFT, self).__init__() - self.n_mel_channels = n_mel_channels - self.sampling_rate = sampling_rate - self.stft_fn = STFT(filter_length, hop_length, win_length) - mel_basis = librosa_mel_fn( - sampling_rate, filter_length, n_mel_channels, mel_fmin, mel_fmax) - mel_basis = torch.from_numpy(mel_basis).float() - self.register_buffer('mel_basis', mel_basis) - - def spectral_normalize(self, magnitudes): - output = dynamic_range_compression(magnitudes) - return output - - def spectral_de_normalize(self, magnitudes): - output = dynamic_range_decompression(magnitudes) - return output - - def mel_spectrogram(self, y): - """Computes mel-spectrograms from a batch of waves - PARAMS - ------ - y: Variable(torch.FloatTensor) with shape (B, T) in range [-1, 1] - - RETURNS - ------- - mel_output: torch.FloatTensor of shape (B, n_mel_channels, T) - """ - assert(torch.min(y.data) >= -1) - assert(torch.max(y.data) <= 1) - - magnitudes, phases = self.stft_fn.transform(y) - magnitudes = magnitudes.data - mel_output = torch.matmul(self.mel_basis, magnitudes) - mel_output = self.spectral_normalize(mel_output) - return mel_output diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/tacotron2/model.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/tacotron2/model.py deleted file mode 100644 index 96a258b99506862deac35c327557e7a40186cde1..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/tacotron2/model.py +++ /dev/null @@ -1,671 +0,0 @@ -from math import sqrt - -import torch -import torch.distributions as distr -from torch import nn -from torch.autograd import Variable -from torch.nn import functional as F - -from .layers import ConvNorm, GlobalAvgPool, LinearNorm -from .utils import get_mask_from_lengths, to_gpu - - -class LocationLayer(nn.Module): - def __init__(self, attention_n_filters, attention_kernel_size, - attention_dim): - super(LocationLayer, self).__init__() - padding = int((attention_kernel_size - 1) / 2) - self.location_conv = ConvNorm(2, attention_n_filters, - kernel_size=attention_kernel_size, - padding=padding, bias=False, stride=1, - dilation=1) - self.location_dense = LinearNorm(attention_n_filters, attention_dim, - bias=False, w_init_gain='tanh') - - def forward(self, attention_weights_cat): - processed_attention = self.location_conv(attention_weights_cat) - processed_attention = processed_attention.transpose(1, 2) - processed_attention = self.location_dense(processed_attention) - return processed_attention - - -class Attention(nn.Module): - def __init__(self, attention_rnn_dim, embedding_dim, attention_dim, - attention_location_n_filters, attention_location_kernel_size): - super(Attention, self).__init__() - self.query_layer = LinearNorm(attention_rnn_dim, attention_dim, - bias=False, w_init_gain='tanh') - self.memory_layer = LinearNorm(embedding_dim, attention_dim, bias=False, - w_init_gain='tanh') - self.v = LinearNorm(attention_dim, 1, bias=False) - self.location_layer = LocationLayer(attention_location_n_filters, - attention_location_kernel_size, - attention_dim) - self.score_mask_value = -float("inf") - - def get_alignment_energies(self, query, processed_memory, - attention_weights_cat): - """ - PARAMS - ------ - query: decoder output (batch, n_mel_channels * n_frames_per_step) - processed_memory: processed encoder outputs (B, T_in, attention_dim) - attention_weights_cat: cumulative and prev. att weights (B, 2, max_time) - - RETURNS - ------- - alignment (batch, max_time) - """ - - processed_query = self.query_layer(query.unsqueeze(1)) - processed_attention_weights = self.location_layer(attention_weights_cat) - energies = self.v(torch.tanh( - processed_query + processed_attention_weights + processed_memory)) - - energies = energies.squeeze(-1) - return energies - - def forward(self, attention_hidden_state, memory, processed_memory, - attention_weights_cat, mask): - """ - PARAMS - ------ - attention_hidden_state: attention rnn last output - memory: encoder outputs - processed_memory: processed encoder outputs - attention_weights_cat: previous and cummulative attention weights - mask: binary mask for padded data - """ - alignment = self.get_alignment_energies( - attention_hidden_state, processed_memory, attention_weights_cat) - - if mask is not None: - alignment.data.masked_fill_(mask, self.score_mask_value) - - attention_weights = F.softmax(alignment, dim=1) - attention_context = torch.bmm(attention_weights.unsqueeze(1), memory) - attention_context = attention_context.squeeze(1) - - return attention_context, attention_weights - - -class Prenet(nn.Module): - def __init__(self, in_dim, sizes): - super(Prenet, self).__init__() - in_sizes = [in_dim] + sizes[:-1] - self.layers = nn.ModuleList( - [LinearNorm(in_size, out_size, bias=False) - for (in_size, out_size) in zip(in_sizes, sizes)]) - - def forward(self, x): - for linear in self.layers: - x = F.dropout(F.relu(linear(x)), p=0.5, training=True) - return x - - -class Postnet(nn.Module): - """Postnet - - Five 1-d convolution with 512 channels and kernel size 5 - """ - - def __init__(self, hparams): - super(Postnet, self).__init__() - self.convolutions = nn.ModuleList() - - self.convolutions.append( - nn.Sequential( - ConvNorm(hparams.n_mel_channels, hparams.postnet_embedding_dim, - kernel_size=hparams.postnet_kernel_size, stride=1, - padding=int((hparams.postnet_kernel_size - 1) / 2), - dilation=1, w_init_gain='tanh'), - nn.BatchNorm1d(hparams.postnet_embedding_dim)) - ) - - for i in range(1, hparams.postnet_n_convolutions - 1): - self.convolutions.append( - nn.Sequential( - ConvNorm(hparams.postnet_embedding_dim, - hparams.postnet_embedding_dim, - kernel_size=hparams.postnet_kernel_size, stride=1, - padding=int((hparams.postnet_kernel_size - 1) / 2), - dilation=1, w_init_gain='tanh'), - nn.BatchNorm1d(hparams.postnet_embedding_dim)) - ) - - self.convolutions.append( - nn.Sequential( - ConvNorm(hparams.postnet_embedding_dim, hparams.n_mel_channels, - kernel_size=hparams.postnet_kernel_size, stride=1, - padding=int((hparams.postnet_kernel_size - 1) / 2), - dilation=1, w_init_gain='linear'), - nn.BatchNorm1d(hparams.n_mel_channels)) - ) - - def forward(self, x): - for i in range(len(self.convolutions) - 1): - x = F.dropout(torch.tanh(self.convolutions[i](x)), 0.5, self.training) - x = F.dropout(self.convolutions[-1](x), 0.5, self.training) - - return x - - -class Encoder(nn.Module): - """Encoder module: - - Three 1-d convolution banks - - Bidirectional LSTM - """ - def __init__(self, hparams): - super(Encoder, self).__init__() - - convolutions = [] - for _ in range(hparams.encoder_n_convolutions): - conv_layer = nn.Sequential( - ConvNorm(hparams.encoder_embedding_dim, - hparams.encoder_embedding_dim, - kernel_size=hparams.encoder_kernel_size, stride=1, - padding=int((hparams.encoder_kernel_size - 1) / 2), - dilation=1, w_init_gain='relu'), - nn.BatchNorm1d(hparams.encoder_embedding_dim)) - convolutions.append(conv_layer) - self.convolutions = nn.ModuleList(convolutions) - - self.lstm = nn.LSTM(hparams.encoder_embedding_dim, - int(hparams.encoder_embedding_dim / 2), 1, - batch_first=True, bidirectional=True) - - def forward(self, x, input_lengths): - for conv in self.convolutions: - x = F.dropout(F.relu(conv(x)), 0.5, self.training) - - x = x.transpose(1, 2) - - # pytorch tensor are not reversible, hence the conversion - input_lengths = input_lengths.cpu().numpy() - x = nn.utils.rnn.pack_padded_sequence( - x, input_lengths, batch_first=True) - - self.lstm.flatten_parameters() - outputs, _ = self.lstm(x) - - outputs, _ = nn.utils.rnn.pad_packed_sequence( - outputs, batch_first=True) - - return outputs - - def inference(self, x): - for conv in self.convolutions: - x = F.dropout(F.relu(conv(x)), 0.5, self.training) - - x = x.transpose(1, 2) - - self.lstm.flatten_parameters() - outputs, _ = self.lstm(x) - - return outputs - - -class AudioEncoder(nn.Module): - def __init__(self, hparams): - super(AudioEncoder, self).__init__() - - assert hparams.lat_dim > 0 - - convolutions = [] - inp_dim = hparams.n_mel_channels - for _ in range(hparams.lat_n_convolutions): - conv_layer = nn.Sequential( - ConvNorm(inp_dim, hparams.lat_n_filters, - kernel_size=hparams.lat_kernel_size, stride=1, - padding=int((hparams.lat_kernel_size - 1) / 2), - dilation=1, w_init_gain='tanh'), - nn.BatchNorm1d(hparams.lat_n_filters)) - inp_dim = hparams.lat_n_filters - convolutions.append(conv_layer) - self.convolutions = nn.ModuleList(convolutions) - - self.lstm = nn.LSTM(hparams.lat_n_filters, - int(hparams.lat_n_filters / 2), - hparams.lat_n_blstms, batch_first=True, - bidirectional=True) - self.pool = GlobalAvgPool() - - self.mu_proj = LinearNorm(hparams.lat_n_filters, hparams.lat_dim) - self.logvar_proj = LinearNorm(hparams.lat_n_filters, hparams.lat_dim) - self.lat_dim = hparams.lat_dim - - def forward(self, x, lengths): - """ - Args: - x (torch.Tensor): (B, F, T) - """ - - for conv in self.convolutions: - x = F.dropout(F.tanh(conv(x)), 0.5, self.training) - - x = x.transpose(1, 2) # (B, T, D) - - # x may not be sorted by length. Sort->process->unsort - max_len = x.size(1) - assert max_len == torch.max(lengths).item() - - lengths, perm_idx = lengths.sort(0, descending=True) - x = x[perm_idx] - x = nn.utils.rnn.pack_padded_sequence(x, lengths, batch_first=True) - - self.lstm.flatten_parameters() - outputs, _ = self.lstm(x) - outputs, _ = nn.utils.rnn.pad_packed_sequence(outputs, batch_first=True) - - _, unperm_idx = perm_idx.sort(0) - outputs = outputs[unperm_idx] # (B, T, D) - lengths = lengths[unperm_idx] # (B, T, D) - - outputs = self.pool(outputs, lengths) # (B, D) - - mu = self.mu_proj(outputs) - logvar = self.logvar_proj(outputs) - z = distr.Normal(mu, logvar).rsample() - return z, mu, logvar - - -class Decoder(nn.Module): - def __init__(self, hparams): - super(Decoder, self).__init__() - self.n_mel_channels = hparams.n_mel_channels - self.n_frames_per_step = hparams.n_frames_per_step - self.encoder_embedding_dim = hparams.encoder_embedding_dim - self.obs_dim = hparams.obs_dim - self.lat_dim = hparams.lat_dim - self.attention_rnn_dim = hparams.attention_rnn_dim - self.decoder_rnn_dim = hparams.decoder_rnn_dim - self.prenet_dim = hparams.prenet_dim - self.max_decoder_steps = hparams.max_decoder_steps - self.gate_threshold = hparams.gate_threshold - self.p_attention_dropout = hparams.p_attention_dropout - self.p_decoder_dropout = hparams.p_decoder_dropout - - self.prenet = Prenet( - hparams.n_mel_channels * hparams.n_frames_per_step, - [hparams.prenet_dim, hparams.prenet_dim]) - - self.attention_rnn = nn.LSTMCell( - hparams.prenet_dim + hparams.encoder_embedding_dim, - hparams.attention_rnn_dim) - - self.attention_layer = Attention( - hparams.attention_rnn_dim, hparams.encoder_embedding_dim, - hparams.attention_dim, hparams.attention_location_n_filters, - hparams.attention_location_kernel_size) - - encoder_tot_dim = (hparams.encoder_embedding_dim + \ - hparams.lat_dim + hparams.obs_dim) - self.decoder_rnn = nn.LSTMCell( - hparams.attention_rnn_dim + encoder_tot_dim, - hparams.decoder_rnn_dim, 1) - - self.linear_projection = LinearNorm( - hparams.decoder_rnn_dim + encoder_tot_dim, - hparams.n_mel_channels * hparams.n_frames_per_step) - - self.gate_layer = LinearNorm( - hparams.decoder_rnn_dim + encoder_tot_dim, 1, - bias=True, w_init_gain='sigmoid') - - def get_go_frame(self, memory): - """ Gets all zeros frames to use as first decoder input - PARAMS - ------ - memory: decoder outputs - - RETURNS - ------- - decoder_input: all zeros frames - """ - B = memory.size(0) - decoder_input = Variable(memory.data.new( - B, self.n_mel_channels * self.n_frames_per_step).zero_()) - return decoder_input - - def initialize_decoder_states(self, memory, obs_and_lat, mask): - """ Initializes attention rnn states, decoder rnn states, attention - weights, attention cumulative weights, attention context, stores memory - and stores processed memory - PARAMS - ------ - memory: Encoder outputs - obs_and_lat: Observed and latent attribute embeddings - mask: Mask for padded data if training, expects None for inference - """ - B = memory.size(0) - MAX_TIME = memory.size(1) - - self.attention_hidden = Variable(memory.data.new( - B, self.attention_rnn_dim).zero_()) - self.attention_cell = Variable(memory.data.new( - B, self.attention_rnn_dim).zero_()) - - self.decoder_hidden = Variable(memory.data.new( - B, self.decoder_rnn_dim).zero_()) - self.decoder_cell = Variable(memory.data.new( - B, self.decoder_rnn_dim).zero_()) - - self.attention_weights = Variable(memory.data.new( - B, MAX_TIME).zero_()) - self.attention_weights_cum = Variable(memory.data.new( - B, MAX_TIME).zero_()) - self.attention_context = Variable(memory.data.new( - B, self.encoder_embedding_dim).zero_()) - - self.memory = memory - self.processed_memory = self.attention_layer.memory_layer(memory) - self.obs_and_lat = obs_and_lat - self.mask = mask - - def parse_decoder_inputs(self, decoder_inputs): - """ Prepares decoder inputs, i.e. mel outputs - PARAMS - ------ - decoder_inputs: inputs used for teacher-forced training, i.e. mel-specs - - RETURNS - ------- - inputs: processed decoder inputs - - """ - # (B, n_mel_channels, T_out) -> (B, T_out, n_mel_channels) - decoder_inputs = decoder_inputs.transpose(1, 2) - decoder_inputs = decoder_inputs.view( - decoder_inputs.size(0), - int(decoder_inputs.size(1)/self.n_frames_per_step), -1) - # (B, T_out, n_mel_channels) -> (T_out, B, n_mel_channels) - decoder_inputs = decoder_inputs.transpose(0, 1) - return decoder_inputs - - def parse_decoder_outputs(self, mel_outputs, gate_outputs, alignments): - """ Prepares decoder outputs for output - PARAMS - ------ - mel_outputs: - gate_outputs: gate output energies - alignments: - - RETURNS - ------- - mel_outputs: - gate_outpust: gate output energies - alignments: - """ - # (T_out, B) -> (B, T_out) - alignments = torch.stack(alignments).transpose(0, 1) - # (T_out, B) -> (B, T_out) - gate_outputs = torch.stack(gate_outputs).transpose(0, 1) - gate_outputs = gate_outputs.contiguous() - # (T_out, B, n_mel_channels) -> (B, T_out, n_mel_channels) - mel_outputs = torch.stack(mel_outputs).transpose(0, 1).contiguous() - # decouple frames per step - mel_outputs = mel_outputs.view( - mel_outputs.size(0), -1, self.n_mel_channels) - # (B, T_out, n_mel_channels) -> (B, n_mel_channels, T_out) - mel_outputs = mel_outputs.transpose(1, 2) - - return mel_outputs, gate_outputs, alignments - - def decode(self, decoder_input): - """ Decoder step using stored states, attention and memory - PARAMS - ------ - decoder_input: previous mel output - - RETURNS - ------- - mel_output: - gate_output: gate output energies - attention_weights: - """ - cell_input = torch.cat((decoder_input, self.attention_context), -1) - self.attention_hidden, self.attention_cell = self.attention_rnn( - cell_input, (self.attention_hidden, self.attention_cell)) - self.attention_hidden = F.dropout( - self.attention_hidden, self.p_attention_dropout, self.training) - - attention_weights_cat = torch.cat( - (self.attention_weights.unsqueeze(1), - self.attention_weights_cum.unsqueeze(1)), dim=1) - self.attention_context, self.attention_weights = self.attention_layer( - self.attention_hidden, self.memory, self.processed_memory, - attention_weights_cat, self.mask) - - self.attention_weights_cum += self.attention_weights - decoder_input = torch.cat( - (self.attention_hidden, self.attention_context), -1) - if self.obs_and_lat is not None: - decoder_input = torch.cat((decoder_input, self.obs_and_lat), -1) - self.decoder_hidden, self.decoder_cell = self.decoder_rnn( - decoder_input, (self.decoder_hidden, self.decoder_cell)) - self.decoder_hidden = F.dropout( - self.decoder_hidden, self.p_decoder_dropout, self.training) - - decoder_hidden_attention_context = torch.cat( - (self.decoder_hidden, self.attention_context), dim=1) - if self.obs_and_lat is not None: - decoder_hidden_attention_context = torch.cat( - (decoder_hidden_attention_context, self.obs_and_lat), dim=1) - decoder_output = self.linear_projection( - decoder_hidden_attention_context) - - gate_prediction = self.gate_layer(decoder_hidden_attention_context) - return decoder_output, gate_prediction, self.attention_weights - - def forward(self, memory, obs_and_lat, decoder_inputs, memory_lengths): - """ Decoder forward pass for training - PARAMS - ------ - memory: Encoder outputs - obs_and_lat: Observed and latent attribute embeddings - decoder_inputs: Decoder inputs for teacher forcing. i.e. mel-specs - memory_lengths: Encoder output lengths for attention masking. - - RETURNS - ------- - mel_outputs: mel outputs from the decoder - gate_outputs: gate outputs from the decoder - alignments: sequence of attention weights from the decoder - """ - - decoder_input = self.get_go_frame(memory).unsqueeze(0) - decoder_inputs = self.parse_decoder_inputs(decoder_inputs) - decoder_inputs = torch.cat((decoder_input, decoder_inputs), dim=0) - decoder_inputs = self.prenet(decoder_inputs) - - self.initialize_decoder_states( - memory, obs_and_lat, mask=~get_mask_from_lengths(memory_lengths)) - - mel_outputs, gate_outputs, alignments = [], [], [] - while len(mel_outputs) < decoder_inputs.size(0) - 1: - decoder_input = decoder_inputs[len(mel_outputs)] - mel_output, gate_output, attention_weights = self.decode( - decoder_input) - mel_outputs += [mel_output.squeeze(1)] - gate_outputs += [gate_output.squeeze()] - alignments += [attention_weights] - - mel_outputs, gate_outputs, alignments = self.parse_decoder_outputs( - mel_outputs, gate_outputs, alignments) - - return mel_outputs, gate_outputs, alignments - - def inference(self, memory, obs_and_lat, ret_has_eos=False): - """ Decoder inference - PARAMS - ------ - memory: Encoder outputs - obs_and_lat: Observed and latent attribute embeddings - - RETURNS - ------- - mel_outputs: mel outputs from the decoder - gate_outputs: gate outputs from the decoder - alignments: sequence of attention weights from the decoder - """ - decoder_input = self.get_go_frame(memory) - - self.initialize_decoder_states(memory, obs_and_lat, mask=None) - - mel_outputs, gate_outputs, alignments = [], [], [] - has_eos = False - while True: - decoder_input = self.prenet(decoder_input) - mel_output, gate_output, alignment = self.decode(decoder_input) - - mel_outputs += [mel_output.squeeze(1)] - gate_outputs += [gate_output] - alignments += [alignment] - - if torch.sigmoid(gate_output.data) > self.gate_threshold: - has_eos = True - break - elif len(mel_outputs) == self.max_decoder_steps: - # print("Warning! Reached max decoder steps") - break - - decoder_input = mel_output - - mel_outputs, gate_outputs, alignments = self.parse_decoder_outputs( - mel_outputs, gate_outputs, alignments) - - if ret_has_eos: - return mel_outputs, gate_outputs, alignments, has_eos - else: - return mel_outputs, gate_outputs, alignments - - -class Tacotron2(nn.Module): - def __init__(self, hparams): - super(Tacotron2, self).__init__() - self.mask_padding = hparams.mask_padding - self.fp16_run = hparams.fp16_run - self.n_mel_channels = hparams.n_mel_channels - self.n_frames_per_step = hparams.n_frames_per_step - - # initialize text encoder embedding - self.embedding = nn.Embedding( - hparams.n_symbols, hparams.symbols_embedding_dim) - std = sqrt(2.0 / (hparams.n_symbols + hparams.symbols_embedding_dim)) - val = sqrt(3.0) * std # uniform bounds for std - self.embedding.weight.data.uniform_(-val, val) - - # initialize observed attribute embedding - self.obs_embedding = None - if hparams.obs_dim > 0: - self.obs_embedding = nn.Embedding( - hparams.obs_n_class, hparams.obs_dim) - std = sqrt(2.0 / (hparams.obs_n_class + hparams.obs_dim)) - val = sqrt(3.0) * std # uniform bounds for std - self.obs_embedding.weight.data.uniform_(-val, val) - - self.encoder = Encoder(hparams) - self.decoder = Decoder(hparams) - self.postnet = Postnet(hparams) - - self.lat_encoder = None - if hparams.lat_dim > 0: - self.lat_encoder = AudioEncoder(hparams) - - def parse_batch(self, batch): - (text_padded, input_lengths, obs_labels, - mel_padded, gate_padded, output_lengths) = batch - text_padded = to_gpu(text_padded).long() - input_lengths = to_gpu(input_lengths).long() - obs_labels = to_gpu(obs_labels).long() - max_len = torch.max(input_lengths.data).item() - mel_padded = to_gpu(mel_padded).float() - gate_padded = to_gpu(gate_padded).float() - output_lengths = to_gpu(output_lengths).long() - - return ( - (text_padded, input_lengths, obs_labels, - mel_padded, max_len, output_lengths), - (mel_padded, gate_padded)) - - def parse_output(self, outputs, output_lengths=None): - if self.mask_padding and output_lengths is not None: - mask = ~get_mask_from_lengths(output_lengths) - mask = mask.expand(self.n_mel_channels, mask.size(0), mask.size(1)) - mask = mask.permute(1, 0, 2) - - outputs[0].data.masked_fill_(mask, 0.0) - outputs[1].data.masked_fill_(mask, 0.0) - outputs[2].data.masked_fill_(mask[:, 0, :], 1e3) # gate energies - - return outputs - - def forward(self, inputs): - (text_inputs, text_lengths, obs_labels, - mels, max_len, output_lengths) = inputs - text_lengths, output_lengths = text_lengths.data, output_lengths.data - - embedded_inputs = self.embedding(text_inputs).transpose(1, 2) - - encoder_outputs = self.encoder(embedded_inputs, text_lengths) - - obs = None - if self.obs_embedding is not None: - obs = self.obs_embedding(obs_labels) - - lat, lat_mu, lat_logvar = None, None, None - if self.lat_encoder is not None: - (lat, lat_mu, lat_logvar) = self.lat_encoder(mels, output_lengths) - - obs_and_lat = [x for x in [obs, lat] if x is not None] - if bool(obs_and_lat): - obs_and_lat = torch.cat(obs_and_lat, dim=-1) - else: - obs_and_lat = None - - mel_outputs, gate_outputs, alignments = self.decoder( - encoder_outputs, obs_and_lat, mels, memory_lengths=text_lengths) - - mel_outputs_postnet = self.postnet(mel_outputs) - mel_outputs_postnet = mel_outputs + mel_outputs_postnet - - return self.parse_output( - [mel_outputs, mel_outputs_postnet, gate_outputs, alignments, - lat_mu, lat_logvar], - output_lengths) - - def inference(self, inputs, obs_labels=None, lat=None, ret_has_eos=False): - embedded_inputs = self.embedding(inputs).transpose(1, 2) - encoder_outputs = self.encoder.inference(embedded_inputs) - - if obs_labels is None: - obs_labels = torch.LongTensor(len(inputs)) - obs_labels = obs_labels.to(inputs.device).zero_() - - obs = None - if self.obs_embedding is not None: - obs = self.obs_embedding(obs_labels) - - if self.lat_encoder is not None: - if lat is None: - lat = torch.FloatTensor(len(inputs), self.lat_encoder.lat_dim) - lat = lat.to(inputs.device).zero_().type(encoder_outputs.type()) - - obs_and_lat = [x for x in [obs, lat] if x is not None] - if bool(obs_and_lat): - obs_and_lat = torch.cat(obs_and_lat, dim=-1) - else: - obs_and_lat = None - - mel_outputs, gate_outputs, alignments, has_eos = self.decoder.inference( - encoder_outputs, obs_and_lat, ret_has_eos=True) - - mel_outputs_postnet = self.postnet(mel_outputs) - mel_outputs_postnet = mel_outputs + mel_outputs_postnet - - outputs = self.parse_output( - [mel_outputs, mel_outputs_postnet, gate_outputs, alignments]) - - if ret_has_eos: - return outputs + [has_eos] - else: - return outputs diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/tacotron2/numbers.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/tacotron2/numbers.py deleted file mode 100644 index c5a2eaf048c54077a4176eb8aed9a3ba034fab0f..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/tacotron2/numbers.py +++ /dev/null @@ -1,71 +0,0 @@ -""" from https://github.com/keithito/tacotron """ - -import re - -import inflect - -_inflect = inflect.engine() -_comma_number_re = re.compile(r'([0-9][0-9\,]+[0-9])') -_decimal_number_re = re.compile(r'([0-9]+\.[0-9]+)') -_pounds_re = re.compile(r'£([0-9\,]*[0-9]+)') -_dollars_re = re.compile(r'\$([0-9\.\,]*[0-9]+)') -_ordinal_re = re.compile(r'[0-9]+(st|nd|rd|th)') -_number_re = re.compile(r'[0-9]+') - - -def _remove_commas(m): - return m.group(1).replace(',', '') - - -def _expand_decimal_point(m): - return m.group(1).replace('.', ' point ') - - -def _expand_dollars(m): - match = m.group(1) - parts = match.split('.') - if len(parts) > 2: - return match + ' dollars' # Unexpected format - dollars = int(parts[0]) if parts[0] else 0 - cents = int(parts[1]) if len(parts) > 1 and parts[1] else 0 - if dollars and cents: - dollar_unit = 'dollar' if dollars == 1 else 'dollars' - cent_unit = 'cent' if cents == 1 else 'cents' - return '%s %s, %s %s' % (dollars, dollar_unit, cents, cent_unit) - elif dollars: - dollar_unit = 'dollar' if dollars == 1 else 'dollars' - return '%s %s' % (dollars, dollar_unit) - elif cents: - cent_unit = 'cent' if cents == 1 else 'cents' - return '%s %s' % (cents, cent_unit) - else: - return 'zero dollars' - - -def _expand_ordinal(m): - return _inflect.number_to_words(m.group(0)) - - -def _expand_number(m): - num = int(m.group(0)) - if num > 1000 and num < 3000: - if num == 2000: - return 'two thousand' - elif num > 2000 and num < 2010: - return 'two thousand ' + _inflect.number_to_words(num % 100) - elif num % 100 == 0: - return _inflect.number_to_words(num // 100) + ' hundred' - else: - return _inflect.number_to_words(num, andword='', zero='oh', group=2).replace(', ', ' ') - else: - return _inflect.number_to_words(num, andword='') - - -def normalize_numbers(text): - text = re.sub(_comma_number_re, _remove_commas, text) - text = re.sub(_pounds_re, r'\1 pounds', text) - text = re.sub(_dollars_re, _expand_dollars, text) - text = re.sub(_decimal_number_re, _expand_decimal_point, text) - text = re.sub(_ordinal_re, _expand_ordinal, text) - text = re.sub(_number_re, _expand_number, text) - return text diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/tacotron2/stft.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/tacotron2/stft.py deleted file mode 100644 index e97cd0e8fb1cd766ea2b0a0fdd7cddec8b9e7893..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/tacotron2/stft.py +++ /dev/null @@ -1,142 +0,0 @@ -""" -BSD 3-Clause License - -Copyright (c) 2017, Prem Seetharaman -All rights reserved. - -* Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, this - list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - -* Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from this - software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -""" - -import numpy as np -import torch -import torch.nn.functional as F -from librosa.util import pad_center, tiny -from scipy.signal import get_window -from torch.autograd import Variable - -from .audio_processing import window_sumsquare - - -class STFT(torch.nn.Module): - """adapted from Prem Seetharaman's https://github.com/pseeth/pytorch-stft""" - def __init__(self, filter_length=800, hop_length=200, win_length=800, - window='hann'): - super(STFT, self).__init__() - self.filter_length = filter_length - self.hop_length = hop_length - self.win_length = win_length - self.window = window - self.forward_transform = None - scale = self.filter_length / self.hop_length - fourier_basis = np.fft.fft(np.eye(self.filter_length)) - - cutoff = int((self.filter_length / 2 + 1)) - fourier_basis = np.vstack([np.real(fourier_basis[:cutoff, :]), - np.imag(fourier_basis[:cutoff, :])]) - - forward_basis = torch.FloatTensor(fourier_basis[:, None, :]) - inverse_basis = torch.FloatTensor( - np.linalg.pinv(scale * fourier_basis).T[:, None, :]) - - if window is not None: - assert(filter_length >= win_length) - # get window and zero center pad it to filter_length - fft_window = get_window(window, win_length, fftbins=True) - fft_window = pad_center(fft_window, filter_length) - fft_window = torch.from_numpy(fft_window).float() - - # window the bases - forward_basis *= fft_window - inverse_basis *= fft_window - - self.register_buffer('forward_basis', forward_basis.float()) - self.register_buffer('inverse_basis', inverse_basis.float()) - - def transform(self, input_data): - num_batches = input_data.size(0) - num_samples = input_data.size(1) - - self.num_samples = num_samples - - # similar to librosa, reflect-pad the input - input_data = input_data.view(num_batches, 1, num_samples) - input_data = F.pad( - input_data.unsqueeze(1), - (int(self.filter_length / 2), int(self.filter_length / 2), 0, 0), - mode='reflect') - input_data = input_data.squeeze(1) - - forward_transform = F.conv1d( - input_data, - Variable(self.forward_basis, requires_grad=False), - stride=self.hop_length, - padding=0) - - cutoff = int((self.filter_length / 2) + 1) - real_part = forward_transform[:, :cutoff, :] - imag_part = forward_transform[:, cutoff:, :] - - magnitude = torch.sqrt(real_part**2 + imag_part**2) - phase = torch.autograd.Variable( - torch.atan2(imag_part.data, real_part.data)) - - return magnitude, phase - - def inverse(self, magnitude, phase): - recombine_magnitude_phase = torch.cat( - [magnitude*torch.cos(phase), magnitude*torch.sin(phase)], dim=1) - - inverse_transform = F.conv_transpose1d( - recombine_magnitude_phase, - Variable(self.inverse_basis, requires_grad=False), - stride=self.hop_length, - padding=0) - - if self.window is not None: - window_sum = window_sumsquare( - self.window, magnitude.size(-1), hop_length=self.hop_length, - win_length=self.win_length, n_fft=self.filter_length, - dtype=np.float32) - # remove modulation effects - approx_nonzero_indices = torch.from_numpy( - np.where(window_sum > tiny(window_sum))[0]) - window_sum = torch.autograd.Variable( - torch.from_numpy(window_sum), requires_grad=False) - window_sum = window_sum.cuda() if magnitude.is_cuda else window_sum - inverse_transform[:, :, approx_nonzero_indices] /= window_sum[approx_nonzero_indices] - - # scale by hop ratio - inverse_transform *= float(self.filter_length) / self.hop_length - - inverse_transform = inverse_transform[:, :, int(self.filter_length/2):] - inverse_transform = inverse_transform[:, :, :-int(self.filter_length/2):] - - return inverse_transform - - def forward(self, input_data): - self.magnitude, self.phase = self.transform(input_data) - reconstruction = self.inverse(self.magnitude, self.phase) - return reconstruction diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/tacotron2/symbols.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/tacotron2/symbols.py deleted file mode 100644 index 5f0d70fdad92ba4f554d971710b60f2f9e8d9298..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/tacotron2/symbols.py +++ /dev/null @@ -1,18 +0,0 @@ -""" from https://github.com/keithito/tacotron """ - -''' -Defines the set of symbols used in text input to the model. - -The default is a set of ASCII characters that works well for English or text that has been run through Unidecode. For other data, you can modify _characters. See TRAINING_DATA.md for details. ''' -from . import cmudict - -_pad = '_' -_punctuation = '!\'(),.:;? ' -_special = '-' -_letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz' - -# Prepend "@" to ARPAbet symbols to ensure uniqueness (some are the same as uppercase letters): -_arpabet = ['@' + s for s in cmudict.valid_symbols] - -# Export all symbols: -symbols = [_pad] + list(_special) + list(_punctuation) + list(_letters) + _arpabet diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/tacotron2/text.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/tacotron2/text.py deleted file mode 100644 index 07f92a5d4287a719ced2ad58db0b6d0cabaf4ba4..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/tacotron2/text.py +++ /dev/null @@ -1,108 +0,0 @@ -""" from https://github.com/keithito/tacotron """ -import re - -import numpy as np - -from . import cleaners -from .symbols import symbols - -# Mappings from symbol to numeric ID and vice versa: -_symbol_to_id = {s: i for i, s in enumerate(symbols)} -_id_to_symbol = {i: s for i, s in enumerate(symbols)} - -# Regular expression matching text enclosed in curly braces: -_curly_re = re.compile(r'(.*?)\{(.+?)\}(.*)') - -# Special symbols -SOS_TOK = '' -EOS_TOK = '' - -def text_to_sequence(text, cleaner_names): - '''Converts a string of text to a sequence of IDs corresponding to the symbols in the text. - - The text can optionally have ARPAbet sequences enclosed in curly braces embedded - in it. For example, "Turn left on {HH AW1 S S T AH0 N} Street." - - Args: - text: string to convert to a sequence - cleaner_names: names of the cleaner functions to run the text through - - Returns: - List of integers corresponding to the symbols in the text - ''' - sequence = [] - - # Check for curly braces and treat their contents as ARPAbet: - while len(text): - m = _curly_re.match(text) - if not m: - sequence += _symbols_to_sequence(_clean_text(text, cleaner_names)) - break - sequence += _symbols_to_sequence(_clean_text(m.group(1), cleaner_names)) - sequence += _arpabet_to_sequence(m.group(2)) - text = m.group(3) - - return sequence - - -def sample_code_chunk(code, size): - assert(size > 0 and size <= len(code)) - start = np.random.randint(len(code) - size + 1) - end = start + size - return code[start:end], start, end - - -def code_to_sequence(code, code_dict, collapse_code): - if collapse_code: - prev_c = None - sequence = [] - for c in code: - if c in code_dict and c != prev_c: - sequence.append(code_dict[c]) - prev_c = c - else: - sequence = [code_dict[c] for c in code if c in code_dict] - if len(sequence) < 0.95 * len(code): - print('WARNING : over 5%% codes are OOV') - - return sequence - - -def sequence_to_text(sequence): - '''Converts a sequence of IDs back to a string''' - result = '' - for symbol_id in sequence: - if symbol_id in _id_to_symbol: - s = _id_to_symbol[symbol_id] - # Enclose ARPAbet back in curly braces: - if len(s) > 1 and s[0] == '@': - s = '{%s}' % s[1:] - result += s - return result.replace('}{', ' ') - - -def sequence_to_code(sequence, code_dict): - '''Analogous to sequence_to_text''' - id_to_code = {i: c for c, i in code_dict.items()} - return ' '.join([id_to_code[i] for i in sequence]) - - -def _clean_text(text, cleaner_names): - for name in cleaner_names: - cleaner = getattr(cleaners, name) - if not cleaner: - raise Exception('Unknown cleaner: %s' % name) - text = cleaner(text) - return text - - -def _symbols_to_sequence(symbols): - return [_symbol_to_id[s] for s in symbols if _should_keep_symbol(s)] - - -def _arpabet_to_sequence(text): - return _symbols_to_sequence(['@' + s for s in text.split()]) - - -def _should_keep_symbol(s): - return s in _symbol_to_id and s != '_' and s != '~' diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/tacotron2/utils.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/tacotron2/utils.py deleted file mode 100644 index f212c832f0c182af476b38d5a3254d8858927e55..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/tacotron2/utils.py +++ /dev/null @@ -1,173 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import collections -import io -import json -import time - -import librosa -import numpy as np -import soundfile as sf -import torch -from scipy.io.wavfile import read - -from .text import EOS_TOK, SOS_TOK - - -def get_mask_from_lengths(lengths): - max_len = torch.max(lengths).item() - ids = torch.arange(0, max_len, out=torch.cuda.LongTensor(max_len)) - mask = (ids < lengths.unsqueeze(1)) - return mask - - -def load_wav_to_torch(full_path, sr=None): - data, sr = librosa.load(full_path, sr=sr) - data = np.clip(data, -1, 1) # potentially out of [-1, 1] due to resampling - data = data * 32768.0 # match values loaded by scipy - return torch.FloatTensor(data.astype(np.float32)), sr - - -def read_binary_audio(bin_data, tar_sr=None): - """ - read binary audio (`bytes` or `uint8` `numpy.ndarray`) to `float32` - `numpy.ndarray` - - RETURNS: - data (np.ndarray) : audio of shape (n,) or (2, n) - tar_sr (int) : sample rate - """ - data, ori_sr = sf.read(io.BytesIO(bin_data), dtype='float32') - data = data.T - if (tar_sr is not None) and (ori_sr != tar_sr): - data = librosa.resample(data, ori_sr, tar_sr) - else: - tar_sr = ori_sr - data = np.clip(data, -1, 1) - data = data * 32768.0 - return torch.FloatTensor(data.astype(np.float32)), tar_sr - - -def load_filepaths_and_text(filename): - with open(filename, encoding='utf-8') as f: - data = [json.loads(line.rstrip()) for line in f] - return data - - -def to_gpu(x): - x = x.contiguous() - - if torch.cuda.is_available(): - x = x.cuda(non_blocking=True) - return torch.autograd.Variable(x) - - -def load_code_dict(path, add_sos=False, add_eos=False): - if not path: - return {} - - with open(path, 'r') as f: - codes = ['_'] + [line.rstrip() for line in f] # '_' for pad - code_dict = {c: i for i, c in enumerate(codes)} - - if add_sos: - code_dict[SOS_TOK] = len(code_dict) - if add_eos: - code_dict[EOS_TOK] = len(code_dict) - assert(set(code_dict.values()) == set(range(len(code_dict)))) - - return code_dict - - -def load_obs_label_dict(path): - if not path: - return {} - with open(path, 'r') as f: - obs_labels = [line.rstrip() for line in f] - return {c: i for i, c in enumerate(obs_labels)} - - -# A simple timer class inspired from `tnt.TimeMeter` -class CudaTimer: - def __init__(self, keys): - self.keys = keys - self.reset() - - def start(self, key): - s = torch.cuda.Event(enable_timing=True) - s.record() - self.start_events[key].append(s) - return self - - def stop(self, key): - e = torch.cuda.Event(enable_timing=True) - e.record() - self.end_events[key].append(e) - return self - - def reset(self): - self.start_events = collections.defaultdict(list) - self.end_events = collections.defaultdict(list) - self.running_times = collections.defaultdict(float) - self.n = collections.defaultdict(int) - return self - - def value(self): - self._synchronize() - return {k: self.running_times[k] / self.n[k] for k in self.keys} - - def _synchronize(self): - torch.cuda.synchronize() - for k in self.keys: - starts = self.start_events[k] - ends = self.end_events[k] - if len(starts) == 0: - raise ValueError("Trying to divide by zero in TimeMeter") - if len(ends) != len(starts): - raise ValueError("Call stop before checking value!") - time = 0 - for start, end in zip(starts, ends): - time += start.elapsed_time(end) - self.running_times[k] += time * 1e-3 - self.n[k] += len(starts) - self.start_events = collections.defaultdict(list) - self.end_events = collections.defaultdict(list) - - -# Used to measure the time taken for multiple events -class Timer: - def __init__(self, keys): - self.keys = keys - self.n = {} - self.running_time = {} - self.total_time = {} - self.reset() - - def start(self, key): - self.running_time[key] = time.time() - return self - - def stop(self, key): - self.total_time[key] = time.time() - self.running_time[key] - self.n[key] += 1 - self.running_time[key] = None - return self - - def reset(self): - for k in self.keys: - self.total_time[k] = 0 - self.running_time[k] = None - self.n[k] = 0 - return self - - def value(self): - vals = {} - for k in self.keys: - if self.n[k] == 0: - raise ValueError("Trying to divide by zero in TimeMeter") - else: - vals[k] = self.total_time[k] / self.n[k] - return vals diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/tacotron2/waveglow_denoiser.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/tacotron2/waveglow_denoiser.py deleted file mode 100644 index f3bdcb6be8a8be905cc4ab70552020395c869610..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/tacotron2/waveglow_denoiser.py +++ /dev/null @@ -1,41 +0,0 @@ -# import sys -# sys.path.append('tacotron2') -import torch - -from .layers import STFT - - -class Denoiser(torch.nn.Module): - """ Removes model bias from audio produced with waveglow """ - - def __init__(self, waveglow, filter_length=1024, n_overlap=4, - win_length=1024, mode='zeros'): - super(Denoiser, self).__init__() - self.stft = STFT(filter_length=filter_length, - hop_length=int(filter_length/n_overlap), - win_length=win_length).cuda() - if mode == 'zeros': - mel_input = torch.zeros( - (1, 80, 88), - dtype=waveglow.upsample.weight.dtype, - device=waveglow.upsample.weight.device) - elif mode == 'normal': - mel_input = torch.randn( - (1, 80, 88), - dtype=waveglow.upsample.weight.dtype, - device=waveglow.upsample.weight.device) - else: - raise Exception("Mode {} if not supported".format(mode)) - - with torch.no_grad(): - bias_audio = waveglow.infer(mel_input, sigma=0.0).float() - bias_spec, _ = self.stft.transform(bias_audio) - - self.register_buffer('bias_spec', bias_spec[:, :, 0][:, :, None]) - - def forward(self, audio, strength=0.1): - audio_spec, audio_angles = self.stft.transform(audio.cuda().float()) - audio_spec_denoised = audio_spec - self.bias_spec * strength - audio_spec_denoised = torch.clamp(audio_spec_denoised, 0.0) - audio_denoised = self.stft.inverse(audio_spec_denoised, audio_angles) - return audio_denoised diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/tts_data.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/tts_data.py deleted file mode 100644 index 8d46c7ae206346e083556b0d5ebfdc048209814c..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/tts_data.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - - -import numpy as np -import torch -from examples.textless_nlp.gslm.unit2speech.tacotron2.text import ( - EOS_TOK, - SOS_TOK, - code_to_sequence, - text_to_sequence, -) -from examples.textless_nlp.gslm.unit2speech.tacotron2.utils import ( - load_code_dict, -) - - -class TacotronInputDataset: - def __init__(self, hparams, append_str=""): - self.is_text = getattr(hparams, "text_or_code", "text") == "text" - if not self.is_text: - self.code_dict = load_code_dict( - hparams.code_dict, hparams.add_sos, hparams.add_eos - ) - self.code_key = hparams.code_key - self.add_sos = hparams.add_sos - self.add_eos = hparams.add_eos - self.collapse_code = hparams.collapse_code - self.append_str = append_str - - def process_code(self, inp_str): - inp_toks = inp_str.split() - if self.add_sos: - inp_toks = [SOS_TOK] + inp_toks - if self.add_eos: - inp_toks = inp_toks + [EOS_TOK] - return code_to_sequence(inp_toks, self.code_dict, self.collapse_code) - - def process_text(self, inp_str): - return text_to_sequence(inp_str, ["english_cleaners"]) - - def get_tensor(self, inp_str): - # uid, txt, inp_str = self._get_data(idx) - inp_str = inp_str + self.append_str - if self.is_text: - inp_toks = self.process_text(inp_str) - else: - inp_toks = self.process_code(inp_str) - return torch.from_numpy(np.array(inp_toks)).long() - - def __len__(self): - return len(self.data) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/utils.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/utils.py deleted file mode 100644 index 7aced08d38301b98b19e2df7d19f1c61150107bc..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/gslm/unit2speech/utils.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - - -import torch -from examples.textless_nlp.gslm.unit2speech.tacotron2.model import Tacotron2 -from examples.textless_nlp.gslm.unit2speech.tacotron2.waveglow_denoiser import ( - Denoiser, -) - - -def load_quantized_audio_from_file(file_path): - base_fname_batch, quantized_units_batch = [], [] - with open(file_path) as f: - for line in f: - base_fname, quantized_units_str = line.rstrip().split("|") - quantized_units = [int(q) for q in quantized_units_str.split(" ")] - base_fname_batch.append(base_fname) - quantized_units_batch.append(quantized_units) - return base_fname_batch, quantized_units_batch - - -def synthesize_audio(model, waveglow, denoiser, inp, lab=None, strength=0.0): - assert inp.size(0) == 1 - inp = inp.cuda() - if lab is not None: - lab = torch.LongTensor(1).cuda().fill_(lab) - - with torch.no_grad(): - _, mel, _, ali, has_eos = model.inference(inp, lab, ret_has_eos=True) - aud = waveglow.infer(mel, sigma=0.666) - aud_dn = denoiser(aud, strength=strength).squeeze(1) - return mel, aud, aud_dn, has_eos - - -def load_tacotron(tacotron_model_path, max_decoder_steps): - ckpt_dict = torch.load(tacotron_model_path) - hparams = ckpt_dict["hparams"] - hparams.max_decoder_steps = max_decoder_steps - sr = hparams.sampling_rate - model = Tacotron2(hparams) - model.load_state_dict(ckpt_dict["model_dict"]) - model = model.cuda().eval().half() - return model, sr, hparams - - -def load_waveglow(waveglow_path): - waveglow = torch.load(waveglow_path)["model"] - waveglow = waveglow.cuda().eval().half() - for k in waveglow.convinv: - k.float() - denoiser = Denoiser(waveglow) - return waveglow, denoiser diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/README.md b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/README.md deleted file mode 100644 index 596467fe8283a7f1514261151f728f7bd233dbd7..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/README.md +++ /dev/null @@ -1,318 +0,0 @@ -# Text-Free Prosody-Aware Generative Spoken Language Modeling - -This folder contains code and recipes to reproduce results reported in a paper _Text-Free Prosody-Aware Generative Spoken Language Modeling_, -Eugene Kharitonov*, Ann Lee*, Adam Polyak, Yossi Adi, Jade Copet, Kushal Lakhotia, Tu-Anh Nguyen, Morgane Rivière, Abdelrahman Mohamed, Emmanuel Dupoux, Wei-Ning Hsu, 2021. arxiv/2109.03264 [[arxiv]](https://arxiv.org/abs/2109.03264). - -`*` denotes equal contribution. - -You can find demo samples [[here]](https://speechbot.github.io/pgslm/index.html). - -
- If you find this code useful, please consider citing our work using this bibtex - -``` - @misc{Kharitonov2021, - title={Text-Free Prosody-Aware Generative Spoken Language Modeling}, - author={Eugene Kharitonov and Ann Lee and Adam Polyak and Yossi Adi and Jade Copet and Kushal Lakhotia and Tu-Anh Nguyen and Morgane Rivière and Abdelrahman Mohamed and Emmanuel Dupoux and Wei-Ning Hsu}, - year={2021}, - eprint={2109.03264}, - archivePrefix={arXiv}, - primaryClass={cs.CL} -} -``` -
- - -## Additional requirements -Three packages are required in addition to fairseq, they are installable with pip: -```bash -pip install AMFM-decompy SoundFile scipy sklearn torchaudio npy-append-array -``` - -## Data preprocessing - -### Prepare unit pseudo-text transcriptions of the audio -To get unit trascripts of the speech data we rely on the preprocessing steps of [GSLM](https://github.com/pytorch/fairseq/tree/main/examples/textless_nlp/gslm/speech2unit/) work. - -Firstly, we will need to prepare manifest files for the dataset we want to preprocess -``` -mkdir manifests/ -python examples/wav2vec/wav2vec_manifest.py --valid-percent=0.0 $DATA_PATH --dest=manifests/train/ -``` -Next, we need a pre-trained HuBERT-base-ls960 model [[download]](https://dl.fbaipublicfiles.com/hubert/hubert_base_ls960.pt) and a corresponding kmeans-100 quantizer [[download]](https://dl.fbaipublicfiles.com/textless_nlp/gslm/hubert/km100/km.bin). Having those we can quantize the dataset: -``` -python examples/textless_nlp/gslm/speech2unit/clustering/quantize_with_kmeans.py \ - --feature_type hubert \ - --kmeans_model_path km.bin \ - --acoustic_model_path hubert_base_ls960.pt \ - --layer 6 \ - --manifest_path manifests/train/train.tsv \ - --out_quantized_file_path manifests/train/units -``` - -Finally, by running -``` -python examples/textless_nlp/pgslm/scripts/join_units_manifest.py --manifest=manifests/train/train.tsv --units=manifests/train/units --output=train.txt -``` -We will get the training data description `train.txt` in the format that pGSLM expects. The above steps have to be repeated for -dev/test sets. Importantly, we rely on an assumption that the directories are structured as in LibriSpeech, i.e. the file paths follow the -`//.wav` format. - -### Preprocess data for pGSLM -The very first step is to obtain the F0 quantization bins. -Assume the vocoder training manifest is `vocoder_train.txt` (in pGSLM data format prepared with the same process above). -We prepare the quantized F0 from the vocoder training data by running -```sh -bash examples/textless_nlp/pgslm/scripts/prepare_f0_quantization.sh \ - vocoder_train.txt 32 # we use 32 bins in the paper -``` -- ``: sampling rate of the audio files in the manifest -- ``: where to output the output files -- ``: prefix of the output files - -The script will generate -- `.f0_stat.pt`: the speaker-level F0 statistics, which can be used in vocoder training -- `_mean_norm_log_f0_bin.th`: the quantized F0, which should be used in `prepare_data.sh` below - -**Note:** See "Pre-trained models" for the pre-computed speaker-level F0 statistics and quantized F0 bins. We suggest using the pre-computed statistics for the data preparation below in order to take advantage of the pre-trained vocoder for waveform generation. - -Next prepare the pGSLM data. -Assume train/valid/test manifests are `{train,valid,test}.txt`. -Here is an example of how to preprocess data: - -```sh -bash examples/textless_nlp/pgslm/scripts/prepare_data.sh \ - train.txt valid.txt test.txt \ - /_mean_norm_log_f0_bin.th -``` -- ``: discrete unit vocabulary size (we used a kmeans quantizer with the number of units equal to 100 in the example above) -- ``: downsampling rate relative to the waveform (e.g., 320 for HuBERT units) -- ``: sampling rate of the audio files in the manifest -- ``: where to output the preprocessed files - -This will create the dataset json config used for the next section at -`/data_config.json`. - -Note that the example script uses only one thread to compute F0, which can take -_very long_ for preprocessing large datasets. It is suggested to distribute -jobs over multiple nodes/processes with `--nshards=x` and `--rank=z` (where z is -in [1, x]) in `preprocess_f0.py`, and set `--nshards_list=x` in -`prepare_data.py` correspondingly to collect sharded F0 data. - -Now, everything is ready for training a model. - -## Training Multi-Stream Transformer Unit Language Model (MS-TLM) - -Below is an example command that trains Multi-Stream Transformer Language Model (MS-TLM) on a prepared dataset: -```bash -DATASET=data_config.json - -fairseq-train $DATASET \ - --task=speech_unit_modeling \ - --arch="transformer_ulm_tiny" \ - --criterion=speech_unit_lm_criterion \ - --share-decoder-input-output-embed \ - --dropout=0.1 \ - --attention-dropout=0.1 \ - --optimizer="adam" \ - --adam-betas="(0.9, 0.98)" \ - --clip-norm=1.0 \ - --lr=0.0005 \ - --lr-scheduler="inverse_sqrt" \ - --warmup-updates=4000 \ - --warmup-init-lr=1e-07 \ - --tokens-per-sample=3072 \ - --max-tokens=3072 \ - --update-freq=4 \ - --max-epoch=70 \ - --num-workers=0 \ - --skip-invalid-size-inputs-valid-test \ - --loss-weights="1.0;0.5;0.0" \ - --ignore-f0-input \ - --checkpoint-activations \ - --fp16 \ - --max-target-positions=4096 \ - --stream-shifts="1,1" \ - --log-f0 --normalize-f0-mean --interpolate-f0 \ - --ignore-unused-valid-subsets \ - --discrete-duration --discrete-f0 -``` - -Some of the important parameters that are specific to MS-TLM: - * `arch`: specifies the Transformer architecture used. Supported options are: - * `transformer_ulm_tiny` - a tiny model that can be used for debugging; it has 2 layers, 1 attention head, FFN and embedding dimensions of 64, - * `transformer_ulm` - a base model with 6 layers, 8 heads, embedding dimension 512, and FFN dimensionality of 2048, - * `transformer_ulm_big` - the largest model we experiment with in the paper: 12-layer/16 heads, 1024/4096 embedding and FFN dimensions; - * `loss-weights`: this parameter sets importance weights (must be non-negative) for the components of the loss that correspond to unit, duration, and F0 streams. To turn off a component of the loss, its weight has to be set to 0. For instance, to predict only unit stream the parameter should be set to "1;0;0"; - * `stream-shifts`: specifies relative shifts of the two prosodic streams w.r.t. the unit stream (duration and F0, respectively). No shift corresponds to "0,0"; - * `ignore-duration-input`/`ignore-f0-input`: setting these flags would zero-out correpsonding input streams; - * `max-token-duration`: duration values would be max-capped by the specified value; - * `discrete-duration`/`discrete-f0`: whether duration and F0 streams should be quantized; - * `log_f0`, `normalize-f0-mean`, `normalize-f0-std`, `interpolate-f0`: configure how F0 stream is treated. `log_f0` sets up modelling in the log-space, `normalize-f0-mean`/`normalize-f0-std` control per-speaker normalization, and `interpolate-f0` enables F0 interpolation for unvoiced regions where F0 was set to 0, - * `mask-dur-prob`, `mask-f0-prob`, `mask-dur-seg-prob`, `mask-f0-seg-prob`, `mask-unit-seg-prob`, `mask-unit-seg-leng`: this family of parameters sets the probababilities of masking individual steps and spans on each stream as well as lengths of the maked spans. - - -## Pre-trained models -### MS-TLM -Below you can find checkpoints for four best-performing models from the paper (IDs 9..12 in Table 1). These models are trained on Hubert-100 transcripts of the LibriLight-6K dataset. They have the prosody streams shifted by 1 w.r.t. the unit stream. All models predict all three streams (units, duration, and F0), but two -of them only have unit steam in their input. - -| | Continuous prosody | Quantized prosody | -|-------------------|--------------------|-------------------| -| No prosody input | [[download]](https://dl.fbaipublicfiles.com/textless_nlp/pgslm/ulm_checkpoints/continuous_no_prosody_shift_1_1.pt) | [[download]](https://dl.fbaipublicfiles.com/textless_nlp/pgslm/ulm_checkpoints/discrete_no_prosody_shift_1_1.pt) | -| Has prosody input | [[download]](https://dl.fbaipublicfiles.com/textless_nlp/pgslm/ulm_checkpoints/continuous_prosody_shift_1_1.pt) | [[download]](https://dl.fbaipublicfiles.com/textless_nlp/pgslm/ulm_checkpoints/discrete_prosody_shift_1_1.pt)| - -The optimal per-stream sampling temperatures/scaling parameters that we have identified for these models, in the (`T-token, T-duration, T-f0`) format: - -| | Continuous prosody | Quantized prosody | -|-------------------|--------------------|-------------------| -| No prosody input | 0.7, 0.125, 0.0003125| 0.7, 0.25, 0.5 | -| Has prosody input | 0.7, 0.125, 0.00125 | 0.7, 0.25, 0.7 | - -## Vocoder -| Units | Prosody | F0 stats | Checkpoint | Config | -|-------------------|---------|--------------|------------|--------| -| HuBERT-base-ls960, kmeans-100 | [[Quantized 32 bins]](https://dl.fbaipublicfiles.com/textless_nlp/pgslm/vocoder/blizzard2013/mean_norm_log_f0_seg_bin.th) | [[download]](https://dl.fbaipublicfiles.com/textless_nlp/pgslm/vocoder/blizzard2013/f0_stats.pt) | [[download]](https://dl.fbaipublicfiles.com/textless_nlp/pgslm/vocoder/blizzard2013/naive_quant_32_norm_log_seg_hubert/checkpoint.pt) | [[download]](https://dl.fbaipublicfiles.com/textless_nlp/pgslm/vocoder/blizzard2013/naive_quant_32_norm_log_seg_hubert/config.json) | -| HuBERT-base-ls960, kmeans-100 | Continuous | [[download]](https://dl.fbaipublicfiles.com/textless_nlp/pgslm/vocoder/blizzard2013/f0_stats.pt) | [[download]](https://dl.fbaipublicfiles.com/textless_nlp/pgslm/vocoder/blizzard2013/mean_norm_log_f0_hubert/checkpoint.pt) | [[download]](https://dl.fbaipublicfiles.com/textless_nlp/pgslm/vocoder/blizzard2013/mean_norm_log_f0_hubert/config.json) | - - -## Evaluating a trained model -Evaluation is done with the `eval/cont_metrics.py` scripts. As described in the paper, there are several metrics used. - -**Teacher-forced metrics** -```bash -SET=valid -CHECKPOINT_PATH=discrete_prosody_shift_1_1.pt -DATA=data_config.json - -python examples/textless_nlp/pgslm/eval/cont_metrics.py $DATA \ - --metric=teacher_force_everything \ - --path=$CHECKPOINT_PATH \ - --batch-size=16 \ - --fp16 \ - --seed=111 \ - --eval-subset=$SET \ - --f0-discretization-bounds=mean_norm_log_f0_seg_bin.th --dequantize-prosody -``` -(Using this command, our provided `discrete_prosody_shift_1_1.pt` checkpoint should produce `{'token_loss': 1.408..., 'duration_loss': 0.5424..., 'f0_loss': 0.0474...}` on LibriSpeech dev-clean). - -The parameters `--f0-discretization-bounds=mean_norm_log_f0_seg_bin.th --dequantize-prosody` are specific for quantized-prosody models. They signal that the prosody streams must be decoded into the continuous domain before calculating correlation. It is the same `*_mean_norm_log_f0_bin.th` file as we prepared before. -The `mean_norm_log_f0_seg_bin.th` file we used with the pre-trained models can be downloaded [[here]](https://dl.fbaipublicfiles.com/textless_nlp/pgslm/vocoder/blizzard2013/mean_norm_log_f0_seg_bin.th). - - -**Consistency (aka Correlation) metrics** - -The following command estimates correlation between mean values of the F0 stream in the prompt and in the generated continuation (unit and duration steams are fixed). - -```bash -T_F0=0.7 -EXPLOSION=20 -SET=test -CHECKPOINT_PATH=discrete_prosody_shift_1_1.pt -DATA=data_config.json - -python examples/textless_nlp/pgslm/eval/cont_metrics.py $DATA \ - --prefix-length=150 \ - --metric=correlation \ - --path=$CHECKPOINT_PATH \ - --batch-size=16 \ - --fp16 \ - --seed=111 \ - --teacher-force-tokens \ - --teacher-force-duration \ - --min-length=300 \ - --batch-explosion-rate=$EXPLOSION \ - --T-f0=$T_F0 \ - --eval-subset=$SET \ - --f0-discretization-bounds=mean_norm_log_f0_seg_bin.th \ - --dequantize-prosody --n-workers=8 -``` -(Using this command, our provided `discrete_prosody_shift_1_1.pt` checkpoint should produce `{...'F0 corr': 0.315 ..}` on LibriSpeech test-clean). - - * By using flags `--teacher-force-tokens, --teacher-force-duration, --teacher-force-f0` one can calculate correlations along each stream while having other two streams fixed to ground-truth values (or freeze all three streams to get ground-truth correlation values); - * The parameters `T-f0`, `T-duration`, and `T-token` specify per-stream temperatures and, in the case of continuous-valued prosody, scaling parameter of the corresponding Laplace distribution (setting a temperature to 0 will enforce greedy sampling); - * `min-length` filters out sequences that are shorter then 300 duration units (i.e. 6s in the case of Hubert units); - * `prefix-length` specifies that we want to use first 150 duration units are prompt (i.e. 3s in the case of Hubert units) - - -**Correctness (aka Continuation) and Expressiveness (aka Std) metrics** - -By running the following command, we can get minMAE and Std for the log-F0 stream for the model with quantized prosody. -```bash -DATA=data_config.json -EXPLOSION=20 -SET=test -CHECKPOINT_PATH=discrete_prosody_shift_1_1.pt -T_F0=0.7 - -python examples/textless_nlp/pgslm/eval/cont_metrics.py $DATA \ - --prefix-length=150 \ - --metric=continuation \ - --path=$CHECKPOINT_PATH \ - --batch-size=16 \ - --fp16 \ - --seed=111 \ - --batch-explosion-rate=$EXPLOSION \ - --teacher-force-tokens \ - --teacher-force-duration \ - --T-f0=$T_F0 \ - --eval-subset=$SET \ - --f0-discretization-bounds=mean_norm_log_f0_seg_bin.th --dequantize-prosody -``` -(Using this command, our provided `discrete_prosody_shift_1_1.pt` checkpoint should produce `{...'F0 MAE': 0.0772, 'F0 Std': 0.1489...}` on LibriSpeech test-clean). - -Again, by setting `--teacher-force-tokens, --teacher-force-duration, --teacher-force-f0` we can calculate Token BLEU for the token stream (when `--teacher-force-duration` & `--teacher-force-f0` are on) and per-stream min MAE for each prosody stream individually. - -Finally, `cont_metrics.py` allows to specify the number of workers (e.g., `n-workers=8`) which allows to speed up the computation by spreading multiple worker processes -over the available GPUs. - -**Cont Word BLEU** - -We used the code and the evaluation protocol of [(Lakhotia et al., 2021)](https://arxiv.org/abs/2102.01192). - -## Sampling from a trained model - -To get (prompted or not) samples from a trained model it is enough to run `sample.py`: -```bash -CHECKPOINT_PATH=checkpoints/checkpoint_best.pt -DATASET=examples/textless_nlp/pgslm/repro/dataset/data_config.json -python examples/textless_nlp/pgslm/sample/sample.py $DATASET \ - --output=$SAMPLES \ - --path=$CHECKPOINT_PATH \ - --sampling \ - --T-token=0.7 \ - --T-duration=0.25 \ - --T-f0=0.7 \ - --max-length=500 \ - --prefix-length=150 \ - --subset=valid \ - --seed=1 \ - --match-duration \ - --code-type=hubert \ - --batch-explosion-rate=2 -``` - -Some useful parameters: - * `T-token`, `T-duration`, `T-f0` specify sampling temperature for the three streams. Setting a temperature to `0` switches sample to the greedy (argmax) one; - * `prefix-length`: length of the prompt, measured in timesteps (e.g. for Hubert (CPC) each timestep is 20 (10) ms); - * `subset`: which subset of the dataset to use as prompts (can be `train`, `valid`, `test`); - * `teacher-force-tokens`, `teacher-force-duration`, `teacher-force-f0`: if set, at each autoregressive step, ground-truth values replace the produced one; - * `short-curcuit`: replace sampling by ground-truth inputs; - * `match-duration`: forces the produced sample to have the same duration (in time), as the entire sequence (beyond the prompt if there is any); - * `batch-explosion-rate`: number of samples per prompt; - * `f0-discretization-bounds`: path to a file with quantization boundaries. If it is set, F0 values are de-quantized back to the continuous domain - (the model must be a quanized one); - * `max-length` sets the maximal number of segment steps to be produced. - -Note that `sample.py` automatically uses all available GPUs, to avoid that please use environment variable `CUDA_VISIBLE_DEVICES`. - -## Vocoding samples -To generate audios for output from `sample.py` (`$IN_FILE`): -```bash -python examples/textless_nlp/pgslm/generate_waveform.py \ - --in-file=$IN_FILE \ - --vocoder=$VODOER \ - --vocoder-cfg=$VOCODER_CFG \ - --results-path=$RESULTS_PATH -``` -See "Pre-trained model" for `$VOCODER` and `VOCODER_CFG`. diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/data_utils.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/data_utils.py deleted file mode 100644 index f0dc3cabe8130a4751d27a10944d515e72c73cb1..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/data_utils.py +++ /dev/null @@ -1,107 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. - -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import os - -import torch -from tqdm import tqdm - - -class Stat: - def __init__(self, keep_raw=False): - self.x = 0.0 - self.x2 = 0.0 - self.z = 0.0 # z = logx - self.z2 = 0.0 - self.n = 0.0 - self.u = 0.0 - self.keep_raw = keep_raw - self.raw = [] - - def update(self, new_x): - new_z = new_x.log() - - self.x += new_x.sum() - self.x2 += (new_x**2).sum() - self.z += new_z.sum() - self.z2 += (new_z**2).sum() - self.n += len(new_x) - self.u += 1 - - if self.keep_raw: - self.raw.append(new_x) - - @property - def mean(self): - return self.x / self.n - - @property - def std(self): - return (self.x2 / self.n - self.mean**2) ** 0.5 - - @property - def mean_log(self): - return self.z / self.n - - @property - def std_log(self): - return (self.z2 / self.n - self.mean_log**2) ** 0.5 - - @property - def n_frms(self): - return self.n - - @property - def n_utts(self): - return self.u - - @property - def raw_data(self): - assert self.keep_raw, "does not support storing raw data!" - return torch.cat(self.raw) - - -class F0Stat(Stat): - def update(self, new_x): - # assume unvoiced frames are 0 and consider only voiced frames - if new_x is not None: - super().update(new_x[new_x != 0]) - - -def dump_speaker_f0_stat(speaker_to_f0_stat, out_prefix): - path = f"{out_prefix}.f0_stat.pt" - assert not os.path.exists(path) - - d = { - speaker: { - "f0_mean": speaker_to_f0_stat[speaker].mean, - "f0_std": speaker_to_f0_stat[speaker].std, - "logf0_mean": speaker_to_f0_stat[speaker].mean_log, - "logf0_std": speaker_to_f0_stat[speaker].std_log, - } - for speaker in speaker_to_f0_stat - } - torch.save(d, path) - - return d - - -def load_audio_path(path): - audio_paths = [] - with open(path) as f: - for line in f.readlines(): - sample = eval(line.strip()) - audio_paths.append(sample["audio"]) - - return audio_paths - - -def load_f0(f0_dir, nshards): - path_to_f0 = {} - for rank in tqdm(range(1, nshards + 1), desc=f"load f0"): - f0_shard_path = f"{f0_dir}/f0_{rank}_{nshards}.pt" - shard_path_to_f0 = torch.load(f0_shard_path) - path_to_f0.update(shard_path_to_f0) - return path_to_f0 diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/eval/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/eval/__init__.py deleted file mode 100644 index 0e028c26b9abf533d16fa07d00d942ca07bbba61..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/eval/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. - -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/eval/cont_metrics.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/eval/cont_metrics.py deleted file mode 100644 index e8d838343c21dab66c14a615e5ff4af363f00208..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/eval/cont_metrics.py +++ /dev/null @@ -1,728 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import os -import pathlib -import sys -import time -from types import SimpleNamespace - -import numpy as np -import scipy -import torch -import torch.multiprocessing as mp -from fairseq import checkpoint_utils, options, utils -from fairseq.criterions.speech_ulm_criterion import mae_loss, nll_loss -from fairseq.data.codedataset import CodeDataset, ExpressiveCodeDataConfig -from fairseq.dataclass.utils import convert_namespace_to_omegaconf -from fairseq.utils import move_to_cuda -from torch.utils.data import DataLoader, DistributedSampler - -sys.path.append(str(pathlib.Path(__file__).parent.parent.resolve())) - -from inference_dataset import InferenceDataset, explode_batch -from naive_decoder import Naive_F0_Decoder -from sample.sample import FilterNamesDataset, TemperatureDecoder, do_sampling - -try: - from nltk.translate.bleu_score import sentence_bleu -except ImportError: - print("Please install nltk: `pip install --user -U nltk`") - raise - - -@torch.no_grad() -def teacher_force_everything( - args, dataset, model, criterion, tgt_dict, rank, world_size -): - prefix = args.prefix_length - - f0_decoder = None - if args.dequantize_prosody: - assert dataset.discrete_f0 - print("Reporting MAE for a discrete model") - f0_decoder = Naive_F0_Decoder( - args.f0_discretization_bounds, dataset.config.f0_vq_n_units - ).cuda() - - dataset = InferenceDataset( - dataset, - prefix=args.prefix_length, - only_prefix=False, - filter_short=True, - presort_by_length=True, - ) - sampler = ( - None - if world_size == 1 - else DistributedSampler( - dataset, num_replicas=world_size, rank=rank, shuffle=False - ) - ) - dataloader = DataLoader( - dataset, - args.batch_size, - shuffle=False, - collate_fn=dataset.collater, - sampler=sampler, - ) - - total_token_loss, total_duration_loss, total_f0_loss, total_tokens = ( - 0.0, - 0.0, - 0.0, - 0.0, - ) - - i = 0 - for batch in dataloader: - i += 1 - batch = move_to_cuda(batch) - output = model(**batch["net_input"]) - - tokens, durations, f0 = output["token"], output["duration"], output["f0"] - durations, f0 = durations.squeeze(), f0.squeeze() - - token_loss = nll_loss( - tokens[:, prefix - 1 :], - batch["target"][:, prefix - 1 :].contiguous(), - batch["mask"][:, prefix - 1 :].contiguous(), - reduce=True, - ) - - if args.dequantize_prosody: - durations = durations.argmax(dim=-1) - duration_loss = mae_loss( - durations[:, prefix - 1 :].contiguous().float(), - batch["dur_target"][:, prefix - 1 :].contiguous().float(), - batch["dur_mask"][:, prefix - 1 :].contiguous(), - reduce=True, - ) - else: - duration_loss = criterion.dur_loss_fn( - durations[:, prefix - 1 :].contiguous(), - batch["dur_target"][:, prefix - 1 :].contiguous(), - batch["dur_mask"][:, prefix - 1 :].contiguous(), - reduce=True, - ) - - if f0_decoder: - f0 = f0.argmax(dim=-1) - f0 = f0_decoder(f0).squeeze(-1) - - f0_target = batch["raw_f0"] - f0_loss = mae_loss( - f0[:, prefix - 1 :].contiguous(), - f0_target[:, prefix - 1 :].contiguous(), - batch["f0_mask"][:, prefix - 1 :].contiguous(), - reduce=True, - ) - else: - f0_loss = criterion.f0_loss_fn( - f0[:, prefix - 1 :].contiguous(), - batch["f0_target"][:, prefix - 1 :].contiguous(), - batch["f0_mask"][:, prefix - 1 :].contiguous(), - reduce=True, - ) - - n_tokens = (~batch["dur_mask"])[:, prefix - 1 :].sum() - - total_token_loss += token_loss.item() - total_duration_loss += duration_loss.item() - total_f0_loss += f0_loss.item() - - total_tokens += n_tokens.item() - if args.debug and i > 5: - break - - values = torch.tensor([total_token_loss, total_duration_loss, total_f0_loss]) - normalizers = torch.tensor([total_tokens for _ in range(3)]) - - return values, normalizers - - -def get_bleu(produced_tokens, target_tokens, tgt_dict): - assert target_tokens.ndim == 1 - assert produced_tokens.size(1) == target_tokens.size(0) - - # we can have padding due to shifted channels - shift = 0 - for token in reversed(target_tokens.cpu().tolist()): - if token in [tgt_dict.pad(), tgt_dict.eos()]: - shift += 1 - else: - break - target_tokens = target_tokens[:-shift] - produced_tokens = produced_tokens[:, :-shift] - - string_target = tgt_dict.string(target_tokens).split() - string_candidates = [ - tgt_dict.string(produced_tokens[i, :]).split() - for i in range(produced_tokens.size(0)) - ] - - bleu3 = sentence_bleu( - references=string_candidates, - hypothesis=string_target, - weights=(1.0 / 3, 1.0 / 3, 1.0 / 3), - ) - return bleu3 - - -@torch.no_grad() -def continuation(args, dataset, model, criterion, tgt_dict, rank, world_size): - is_discrete_duration = dataset.discrete_dur - is_discrete_f0 = dataset.discrete_f0 - - f0_decoder = None - if args.dequantize_prosody: - assert dataset.discrete_f0 - print("Reporting MAE F0 for a discrete model") - f0_decoder = Naive_F0_Decoder( - args.f0_discretization_bounds, dataset.config.f0_vq_n_units - ).cuda() - - dataset = InferenceDataset( - dataset, args.prefix_length, filter_short=True, presort_by_length=True - ) - sampler = ( - None - if world_size == 1 - else DistributedSampler( - dataset, num_replicas=world_size, rank=rank, shuffle=False - ) - ) - dataloader = DataLoader( - dataset, - batch_size=1, - shuffle=False, - collate_fn=dataset.collater, - sampler=sampler, - ) - - Ts = args.T_token, args.T_duration, args.T_f0 - decoder = TemperatureDecoder( - Ts, discrete_dur=is_discrete_duration, discrete_f0=is_discrete_f0 - ) - - running_stats = SimpleNamespace( - token_bleu=0.0, - duration_nll=0.0, - duration_mae=0.0, - f0_nll=0.0, - f0_mae=0.0, - n_tokens=0.0, - n_sentences=0.0, - f0_sum=0.0, - f0_sum_sq=0.0, - dur_sum=0.0, - dur_sum_sq=0.0, - ) - - for i, batch in enumerate(dataloader): - batch = explode_batch(batch, args.batch_explosion_rate) - bsz = batch["target"].size(0) - - batch = move_to_cuda(batch) - prefix = batch["prefix"][0] - - max_length_to_unroll = batch["target"].size(1) - prefix_length = batch["net_input"]["src_tokens"].size(1) - steps = max_length_to_unroll - prefix_length + 1 - - assert steps > 0 - produced_tokens, produced_durations, produced_f0, outputs = do_sampling( - model, - batch, - tgt_dict.eos(), - decoder, - autoregressive_steps=steps, - teacher_force_tokens=args.teacher_force_tokens, - teacher_force_duration=args.teacher_force_duration, - teacher_force_f0=args.teacher_force_f0, - ) - - if args.teacher_force_tokens: - assert (produced_tokens[:, 1:] == batch["target"]).all() - if args.teacher_force_duration: - assert (produced_durations[:, 1:] == batch["dur_target"]).all() - if args.teacher_force_f0: - assert (produced_f0[:, 1:] == batch["f0_target"]).all() - - dur_target = batch["dur_target"][:, prefix - 1 :].contiguous() - f0_target = batch["f0_target"][:, prefix - 1 :].contiguous() - - f0_mask = batch["f0_mask"][:, prefix - 1 :].contiguous() - dur_mask = batch["dur_mask"][:, prefix - 1 :].contiguous() - - duration_mae = mae_loss( - produced_durations[:, prefix:].float(), - dur_target.float(), - dur_mask, - reduce=False, - ) - min_duration_mae = duration_mae.view(bsz, -1).sum(dim=-1).min(dim=0)[0] - running_stats.duration_mae += min_duration_mae - - running_stats.dur_sum += ( - produced_durations[:, prefix:].float() * (~dur_mask) - ).sum() / args.batch_explosion_rate - running_stats.dur_sum_sq += ( - produced_durations[:, prefix:].float() * (~dur_mask) - ).pow(2.0).sum() / args.batch_explosion_rate - - if is_discrete_duration: - duration_loss = criterion.dur_loss_fn( - torch.stack([x[1] for x in outputs], dim=1), - dur_target, - dur_mask, - reduce=False, - ) - min_duration_loss = duration_loss.view(bsz, -1).sum(dim=-1).min(dim=0)[0] - running_stats.duration_nll += min_duration_loss - - if f0_decoder: # can only exist for discrete F0 models - decoded_produced_f0 = f0_decoder(produced_f0[:, prefix:]) - decoded_f0_target = batch["raw_f0"][:, prefix - 1 :].contiguous() - - if produced_f0.ndim == 3: - decoded_produced_f0 = decoded_produced_f0.squeeze(2) - decoded_f0_target = decoded_f0_target.squeeze(2) - - f0_mae = mae_loss( - decoded_produced_f0, decoded_f0_target, f0_mask, reduce=False - ) - f0_mae = f0_mae.view(bsz, -1).sum(dim=-1).min(dim=0)[0] - running_stats.f0_mae += f0_mae - - f0_loss = criterion.f0_loss_fn( - torch.stack([x[2] for x in outputs], dim=1), - f0_target.long(), - f0_mask, - reduce=False, - ) - f0_loss = f0_loss.view(bsz, -1).sum(dim=-1).min(dim=0)[0] - running_stats.f0_nll += f0_loss - - running_stats.f0_sum += ( - decoded_produced_f0 * (~f0_mask) - ).sum() / args.batch_explosion_rate - running_stats.f0_sum_sq += (decoded_produced_f0 * (~f0_mask)).pow( - 2.0 - ).sum() / args.batch_explosion_rate - - else: - assert not is_discrete_duration - - f0_loss = mae_loss( - produced_f0[:, prefix:], f0_target, f0_mask, reduce=False - ) - f0_loss = f0_loss.view(bsz, -1).sum(dim=-1).min(dim=0)[0] - running_stats.f0_mae += f0_loss - - running_stats.f0_sum += ( - produced_f0[:, prefix:].sum() / args.batch_explosion_rate - ) - running_stats.f0_sum_sq += ( - produced_f0[:, prefix:].pow(2.0).sum() / args.batch_explosion_rate - ) - - running_stats.n_tokens += (~dur_mask)[0, ...].sum() - - token_loss = get_bleu( - produced_tokens[:, prefix:], batch["target"][0, prefix - 1 :], tgt_dict - ) - running_stats.token_bleu += token_loss - running_stats.n_sentences += 1 - - if args.debug: - break - - values = torch.tensor( - [ - running_stats.token_bleu, - running_stats.duration_nll, - running_stats.duration_mae, - running_stats.f0_nll, - running_stats.f0_mae, - running_stats.f0_sum, - running_stats.f0_sum_sq, - running_stats.dur_sum, - running_stats.dur_sum_sq, - ] - ) - normalizers = torch.tensor( - [running_stats.n_sentences] + [running_stats.n_tokens] * 8 - ) - - return values, normalizers - - -@torch.no_grad() -def correlation(args, dataset, model, criterion, tgt_dict, rank, world_size): - is_discrete_duration = dataset.discrete_dur - is_discrete_f0 = dataset.discrete_f0 - - f0_decoder = None - if is_discrete_f0: - assert dataset.discrete_f0 - f0_decoder = Naive_F0_Decoder( - args.f0_discretization_bounds, dataset.config.f0_vq_n_units - ).cuda() - - if is_discrete_f0: - assert f0_decoder # correlation on tokens is meaningless - - dataset = InferenceDataset( - dataset, - args.prefix_length, - filter_short=True, - presort_by_length=True, - min_length=args.min_length, - ) - sampler = ( - None - if world_size == 1 - else DistributedSampler( - dataset, num_replicas=world_size, rank=rank, shuffle=False - ) - ) - dataloader = DataLoader( - dataset, - batch_size=1, - shuffle=False, - collate_fn=dataset.collater, - sampler=sampler, - ) - - Ts = args.T_token, args.T_duration, args.T_f0 - decoder = TemperatureDecoder( - Ts, discrete_dur=is_discrete_duration, discrete_f0=is_discrete_f0 - ) - - mean_dur_prefix, mean_dur_cont = [], [] - mean_f0_prefix, mean_f0_cont = [], [] - - for batch in dataloader: - batch = explode_batch(batch, args.batch_explosion_rate) - batch = move_to_cuda(batch) - - assert len(batch["prefix"]) == 1 - - if args.teacher_force_tokens: - autoregressive_steps = batch["target"].size(1) - args.prefix_length - 1 - else: - autoregressive_steps = args.max_length - args.prefix_length # + max_shift? - - if args.copy_target: - produced_durations, produced_f0 = batch["dur_target"], batch["f0_target"] - else: - _, produced_durations, produced_f0, outputs = do_sampling( - model, - batch, - tgt_dict.eos(), - decoder, - autoregressive_steps=autoregressive_steps, - teacher_force_tokens=args.teacher_force_tokens, - teacher_force_duration=args.teacher_force_duration, - teacher_force_f0=args.teacher_force_f0, - ) - - # first tokens actually correspond to BOS - produced_durations = produced_durations[:, 1:] - produced_f0 = produced_f0[:, 1:] - - dur_target = batch["dur_target"] - if is_discrete_duration: - produced_durations = produced_durations.float() - dur_target = dur_target.float() - - if is_discrete_f0: - produced_f0 = f0_decoder(produced_f0).squeeze(-1) - f0_target = batch["raw_f0"] - else: - f0_target = batch["f0_target"] - - # prefix values - prefix = batch["prefix"][0] - dur_prefix_mean = dur_target[:, :prefix].sum(dim=-1) / ( - (~batch["dur_mask"][:, :prefix]).sum(dim=-1) - ) - - non_voiced = f0_target[:, :prefix] == 0.0 - f0_mask = batch["f0_mask"][:, :prefix].logical_or(non_voiced) - f0_prefix_mean = f0_target[:, :prefix].sum(dim=-1) / ((~f0_mask).sum(dim=-1)) - - # continuation values - dur_cont_mean = produced_durations[:, prefix:].sum(dim=-1) / ( - (~batch["dur_mask"][:, prefix:]).sum(dim=-1) - ) - - non_voiced = produced_f0[:, prefix:] == 0.0 - f0_mask = non_voiced - f0_cont_mean = produced_f0[:, prefix:].sum(dim=-1) / ((~f0_mask).sum(dim=-1)) - - assert not f0_cont_mean.isnan().any() - - mean_dur_prefix.append(dur_prefix_mean.cpu()) - mean_dur_cont.append(dur_cont_mean.cpu()) - - mean_f0_prefix.append(f0_prefix_mean.cpu()) - mean_f0_cont.append(f0_cont_mean.cpu()) - - if args.debug and len(mean_dur_prefix) > 10: - break - - mean_dur_prefix, mean_dur_cont = torch.cat(mean_dur_prefix), torch.cat( - mean_dur_cont - ) - mean_f0_prefix, mean_f0_cont = torch.cat(mean_f0_prefix), torch.cat(mean_f0_cont) - - return mean_dur_prefix, mean_dur_cont, mean_f0_prefix, mean_f0_cont - - -def main(rank, world_size, args): - start = time.time() - - if world_size > 1: - torch.distributed.init_process_group( - backend="gloo", init_method="env://", world_size=world_size, rank=rank - ) - torch.cuda.set_device(rank % torch.cuda.device_count()) - - raw_args = args - - args = convert_namespace_to_omegaconf(args) - if args.common.seed is not None: - np.random.seed(args.common.seed) - utils.set_torch_seed(args.common.seed) - - models, model_args, task = checkpoint_utils.load_model_ensemble_and_task( - [raw_args.path], arg_overrides={"data": args.task.data} - ) - - tgt_dict = task.target_dictionary - - for model in models: - model.prepare_for_inference_(args) - model.cuda().eval() - if raw_args.fp16: - model = model.half() - model = models[0] - - config = ExpressiveCodeDataConfig(args.task.data) - - dataset = CodeDataset( - manifest=config.manifests[raw_args.eval_subset], - dictionary=task.source_dictionary, - dur_dictionary=task.source_duration_dictionary, - f0_dictionary=task.source_f0_dictionary, - config=config, - discrete_dur=task.cfg.discrete_duration, - discrete_f0=task.cfg.discrete_f0, - log_f0=task.cfg.log_f0, - normalize_f0_mean=task.cfg.normalize_f0_mean, - normalize_f0_std=task.cfg.normalize_f0_std, - interpolate_f0=task.cfg.interpolate_f0, - shifts=task.cfg.stream_shifts, - return_filename=True, - strip_filename=False, - return_continuous_f0=raw_args.dequantize_prosody, - ) - - if raw_args.filter_names: - dataset = FilterNamesDataset(dataset, raw_args.filter_names) - - criterion = task.build_criterion(model_args.criterion) - - name2metric = { - "continuation": continuation, - "teacher_force_everything": teacher_force_everything, - "correlation": correlation, - } - - name2keys = { - "continuation": ( - "Token BLEU3", - "Duration NLL", - "Duration MAE", - "F0 NLL", - "F0 MAE", - "F0 sum", - "F0 sum_sq", - "Dur sum", - "Dur sum_sq", - ), - "teacher_force_everything": ("token_loss", "duration_loss", "f0_loss"), - "correlation": ("Duration corr", "F0 corr"), - } - metric_name = raw_args.metric - - metric = name2metric[metric_name] - results = metric(raw_args, dataset, model, criterion, tgt_dict, rank, world_size) - - values = None - - if metric_name not in [ - "correlation", - ]: - values, normalizers = results - values = maybe_aggregate_normalize(values, normalizers, world_size) - elif metric_name == "correlation": - values = maybe_aggregate_correlations(results, world_size) - else: - assert False - - assert values is not None - summary = dict(zip(name2keys[raw_args.metric], values.tolist())) - if metric_name == "continuation": - summary["F0 Std"] = np.sqrt(-summary["F0 sum"] ** 2 + summary["F0 sum_sq"]) - summary["Dur Std"] = np.sqrt(-summary["Dur sum"] ** 2 + summary["Dur sum_sq"]) - del summary["F0 sum"] - del summary["F0 sum_sq"] - del summary["Dur sum"] - del summary["Dur sum_sq"] - - summary["metric"] = metric_name - - if rank == 0: - print(summary) - if raw_args.wandb: - wandb_results(summary, raw_args) - print("# finished in ", time.time() - start, "seconds") - - -def wandb_results(summary, raw_args): - import wandb - - run = wandb.init( - project=raw_args.wandb_project_name, tags=raw_args.wandb_tags.split(",") - ) - run.config.metric = raw_args.metric - run.config.model = raw_args.path - run.config.data = raw_args.data - - if raw_args.wandb_run_name: - run.name = raw_args.wandb_run_name - run.save() - - wandb.log(summary) - wandb.finish() - - -def maybe_aggregate_normalize(values, normalizers, world_size): - if world_size > 1: - torch.distributed.barrier() - - torch.distributed.all_reduce_multigpu([values]) - torch.distributed.all_reduce_multigpu([normalizers]) - - return values / normalizers - - -def maybe_aggregate_correlations(results, world_size): - if world_size > 1: - output = [None for _ in range(world_size)] - torch.distributed.all_gather_object(output, results) - mean_dur_prefix, mean_dur_cont, mean_f0_prefix, mean_f0_cont = [ - torch.cat([x[i] for x in output]) for i in range(4) - ] - else: - mean_dur_prefix, mean_dur_cont, mean_f0_prefix, mean_f0_cont = results - - corr_dur = scipy.stats.pearsonr(mean_dur_prefix.numpy(), mean_dur_cont.numpy())[0] - corr_f0 = scipy.stats.pearsonr(mean_f0_prefix.numpy(), mean_f0_cont.numpy())[0] - values = torch.tensor([corr_dur, corr_f0]) - - return values - - -def cli_main(): - parser = options.get_interactive_generation_parser() - parser.add_argument( - "--prefix-length", - type=int, - default=1, - help="Prompt prefix length (including )", - ) - parser.add_argument( - "--duration-scale", - type=float, - default=1, - help="Multiply durations by the given scaler", - ) - parser.add_argument( - "--debug", action="store_true", help="Process only the first batch" - ) - parser.add_argument("--n_hypotheses", type=int, default=1) - parser.add_argument("--filter-names", type=str, default=None) - parser.add_argument( - "--max-length", type=int, default=200, help="Maximal produced length" - ) - - parser.add_argument("--teacher-force-tokens", action="store_true", default=False) - parser.add_argument("--teacher-force-duration", action="store_true", default=False) - parser.add_argument("--teacher-force-f0", action="store_true", default=False) - - parser.add_argument("--copy-target", action="store_true", default=False) - parser.add_argument("--min-length", type=int, default=None) - parser.add_argument("--f0-discretization-bounds", type=str, default=None) - parser.add_argument("--dequantize-prosody", action="store_true") - parser.add_argument("--batch-explosion-rate", type=int, default=1) - - parser.add_argument( - "--metric", - choices=["continuation", "teacher_force_everything", "correlation"], - required=True, - ) - - parser.add_argument("--wandb", action="store_true") - parser.add_argument("--wandb-project-name", type=str, default="eslm") - parser.add_argument("--wandb-tags", type=str, default="") - parser.add_argument("--wandb-run-name", type=str, default="") - - parser.add_argument("--T-token", type=float, default=1.0) - parser.add_argument("--T-duration", type=float, default=1.0) - parser.add_argument("--T-f0", type=float, default=1.0) - - parser.add_argument("--n-workers", type=int, default=1) - - parser.add_argument( - "--eval-subset", type=str, default="valid", choices=["valid", "test"] - ) - - args = options.parse_args_and_arch(parser) - - assert ( - args.prefix_length >= 1 - ), "Prefix length includes bos token , hence the minimum is 1." - assert args.temperature >= 0.0, "T must be non-negative!" - - if args.dequantize_prosody: - assert args.f0_discretization_bounds - - world_size = args.n_workers or torch.cuda.device_count() - if world_size > 1: - import random - - mp.set_start_method("spawn", force=True) - os.environ["MASTER_ADDR"] = "localhost" - os.environ["MASTER_PORT"] = str(random.randint(10_000, 50_000)) - - mp.spawn( - main, - nprocs=world_size, - args=( - world_size, - args, - ), - join=True, - ) - else: - main(rank=0, world_size=world_size, args=args) - - -if __name__ == "__main__": - cli_main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/generate_waveform.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/generate_waveform.py deleted file mode 100644 index d1d932d26770be178d80a48fdac36c2f2d2f7b94..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/generate_waveform.py +++ /dev/null @@ -1,118 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import ast -import json -import logging -from pathlib import Path - -import soundfile as sf -import torch -from fairseq import utils -from fairseq.models.text_to_speech.vocoder import CodeHiFiGANVocoder -from tqdm import tqdm - -logging.basicConfig() -logging.root.setLevel(logging.INFO) -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - - -def dump_result(args, data, sample_id, pred_wav): - assert "audio" in data or args.results_path is not None - if args.results_path: - fname = Path(data["audio"]).name if "audio" in data else f"{sample_id}_pred.wav" - out_file = Path(args.results_path) / fname - - sf.write( - out_file.as_posix(), - pred_wav.detach().cpu().numpy(), - args.sample_rate, - ) - - -def load_data(in_file): - with open(in_file) as f: - data = [ast.literal_eval(line.strip()) for line in f] - - return data - - -def get_f0_upsample_ratio(code_hop_size, f_hop_size): - ratio = (code_hop_size // 160) // (f_hop_size // 256) * 2 - return ratio - - -def main(args): - logger.info(args) - - use_cuda = torch.cuda.is_available() and not args.cpu - - with open(args.vocoder_cfg) as f: - vocoder_cfg = json.load(f) - vocoder = CodeHiFiGANVocoder(args.vocoder, vocoder_cfg) - if use_cuda: - vocoder = vocoder.cuda() - - data = load_data(args.in_file) - - if args.results_path: - Path(args.results_path).mkdir(exist_ok=True, parents=True) - - for i, d in tqdm(enumerate(data), total=len(data)): - code_key = "cpc_km100" if "cpc_km100" in d else "hubert" - code = list(map(int, d[code_key].split())) - - x = { - "code": torch.LongTensor(code).view(1, -1), - "f0": torch.Tensor(d["f0"]).view(1, -1), - } - - f0_up_ratio = get_f0_upsample_ratio( - vocoder_cfg["code_hop_size"], vocoder_cfg["hop_size"] - ) - if f0_up_ratio > 1: - bsz, cond_length = x["f0"].size() - x["f0"] = x["f0"].unsqueeze(2).repeat(1, 1, f0_up_ratio).view(bsz, -1) - - x = utils.move_to_cuda(x) if use_cuda else x - wav = vocoder(x) - dump_result(args, d, i, wav) - - -def cli_main(): - parser = argparse.ArgumentParser() - parser.add_argument( - "--in-file", - type=str, - required=True, - help="Input file following the same format of the output from sample.py ('f0' and 'cpc_km100/hubert' are required fields)", - ) - parser.add_argument( - "--vocoder", type=str, required=True, help="path to the vocoder" - ) - parser.add_argument( - "--vocoder-cfg", - type=str, - required=True, - help="path to the vocoder config", - ) - parser.add_argument("--sample-rate", type=int, default=16_000) - parser.add_argument( - "--results-path", - type=str, - default=None, - help="Output directory. If not set, the audios will be stored following the 'audio' field specified in the input file.", - ) - parser.add_argument("--cpu", action="store_true", help="run on CPU") - - args = parser.parse_args() - - main(args) - - -if __name__ == "__main__": - cli_main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/inference_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/inference_dataset.py deleted file mode 100644 index 9f7cfa5f5437f17a665070f0c539c11a92e30855..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/inference_dataset.py +++ /dev/null @@ -1,103 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. - -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - - -import torch - - -class InferenceDataset: - def __init__( - self, - dataset, - prefix, - only_prefix=True, - presort_by_length=True, - filter_short=False, - min_length=None, - ): - self.dataset = dataset - self.collater = self.dataset.collater - self.prefix = prefix - self.only_prefix = only_prefix - self.filter_short = filter_short - - self.remapping = list(range(len(self.dataset))) - if min_length: - assert min_length >= prefix + 1 - - length_thr = prefix + 1 if not min_length else min_length - - if filter_short: - self.remapping = list( - filter( - lambda i: self.dataset[i]["dur_source"].sum() > length_thr, - self.remapping, - ) - ) - print( - f"# the initial dataset of {len(self.dataset)} examples became {len(self.remapping)} after filtering" - f" examples shorter than {length_thr} (in duration units)" - ) - - if presort_by_length: - lengths = {index: dataset.size(index) for index in self.remapping} - self.remapping.sort(key=lambda i: lengths[i]) - - @property - def pads(self): - return self.dataset.pads - - def __len__(self): - return len(self.remapping) - - def original_size(self, k): - k = self.remapping[k] - return self.dataset.size(k) - - def __getitem__(self, k): - k = self.remapping[k] - channels = self.dataset[k] - - if self.prefix and self.only_prefix: - dur_channel = channels["dur_source"] - assert dur_channel.sum() >= self.prefix - - token_times = dur_channel.cumsum(dim=-1) - cut_after = torch.searchsorted(token_times, torch.tensor(self.prefix)) - - r = {} - for channel_name, value in channels.items(): - if isinstance(value, torch.Tensor) and "source" in channel_name: - # if self.filter_short: assert value.size(0) >= self.prefix - r[channel_name] = value[: cut_after + 1] - else: - r[channel_name] = value - - r["prefix"] = cut_after + 1 - else: - r = channels - - return r - - -def explode_batch(batch, times): - if times == 1: - return batch - - new_batch = {} - - for key, value in batch.items(): - if isinstance(value, torch.Tensor): - assert value.size(0) == 1 - new_batch[key] = torch.cat([value] * times) - elif key in ["ntokens", "nsentences"]: - new_batch[key] = value * times - elif key in ["prefix", "filename"]: - new_batch[key] = value - elif key == "net_input": - new_batch[key] = explode_batch(value, times) - else: - assert False, key - return new_batch diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/naive_decoder.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/naive_decoder.py deleted file mode 100644 index b52135dbabeb6ec2a02537d3e9ffdf3887fb46ce..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/naive_decoder.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. - -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import warnings - -import torch - - -class Naive_F0_Decoder(torch.nn.Module): - def __init__(self, bounds_path, n_units=32): - super().__init__() - - bounds = torch.load(bounds_path) - bounds = torch.from_numpy(bounds[n_units]) - assert bounds.ndim == 1 - - pad = torch.tensor([-5.0, -5.0]) # bos, eos, pad are in the dictionary - centers = torch.cat( - [bounds[0:1], 0.5 * (bounds[1:] + bounds[:-1]), bounds[-1:], pad[:]] - ) - - self.embedding = torch.nn.Embedding.from_pretrained( - centers.unsqueeze(-1), freeze=True - ) - self.max_n = self.embedding.weight.numel() - - def forward(self, discrete_f0: torch.Tensor): - in_bounds = (0 <= discrete_f0).all() and (discrete_f0 < self.max_n).all() - if not in_bounds: - warnings.warn( - f"F0 contains some weird outputs: discrete_f0.max().item()={discrete_f0.max().item()} discrete_f0.min().item()={discrete_f0.min().item()}; " - f"while we have embeddings for {self.max_n} values. " - "Assuming this is a no-prosody model -- but be careful!" - ) - - mask = discrete_f0 >= self.max_n - discrete_f0 = discrete_f0.masked_fill(mask, self.max_n - 1) - - return self.embedding(discrete_f0).squeeze(-1) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/prepare_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/prepare_dataset.py deleted file mode 100644 index 7a77985feb93d48913299d17ea2733b6b5775cf7..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/prepare_dataset.py +++ /dev/null @@ -1,141 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. - -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import os -from collections import defaultdict -from itertools import starmap -from multiprocessing import Pool - -import torch -from data_utils import F0Stat, dump_speaker_f0_stat, load_f0 -from fairseq.data.codedataset import ( - F0_FRAME_SPACE, - ExpressiveCodeDataConfig, - align_f0_to_durations, - parse_manifest, -) -from fairseq.tasks.speech_ulm_task import UnitDictionary -from npy_append_array import NpyAppendArray -from tqdm import tqdm - - -def load_meta(meta_path, split): - config = ExpressiveCodeDataConfig(meta_path) - manifest_path = config.manifests[split] - dictionary = UnitDictionary(n_units=config.n_units) - audio_paths, codes, durs, speakers = parse_manifest(manifest_path, dictionary) - return config, audio_paths, codes, durs, speakers - - -def _align_f0(f0, dur, ratio, frm_tol=5): - if f0 is None: - seg_f0 = torch.zeros_like(dur, dtype=torch.float) - else: - seg_f0 = align_f0_to_durations(f0, dur, ratio, tol=frm_tol * ratio) - return seg_f0.numpy() # try a hacky stuff - - -def align_f0(path_to_f0, audio_paths, durs, ratio, mp=False): - chunk_size = 2000 - num_procs = 40 - iterable = ((path_to_f0[p], d, ratio) for p, d in zip(audio_paths, durs)) - - seg_f0s = [] - if mp: - with Pool(num_procs) as pool: - iterator = tqdm( - pool.istarmap(_align_f0, iterable, chunk_size), - desc="align f0", - total=len(durs), - ) - for seg_f0 in iterator: - seg_f0s.append(torch.from_numpy(seg_f0).float()) - else: - iterator = tqdm(starmap(_align_f0, iterable), desc="align f0", total=len(durs)) - for seg_f0 in iterator: - seg_f0s.append(torch.from_numpy(seg_f0).float()) - - return seg_f0s - - -def prepare_seg_data(config, audio_paths, codes, durs, speakers, path_to_f0): - ratio = config.code_hop_size / (config.sampling_rate * F0_FRAME_SPACE) - seg_f0s = align_f0(path_to_f0, audio_paths, durs, ratio) - data = { - "codes": codes, - "duration": durs, - "f0": seg_f0s, - "speaker": speakers, - "path": audio_paths, - } - return data - - -def dump_seg_data(data, out_prefix): - key_targs = { - "codes": f"{out_prefix}.code.npy", - "duration": f"{out_prefix}.dur.npy", - "f0": f"{out_prefix}.f0.npy", - } - for key, targ in key_targs.items(): - assert not os.path.exists(targ) - npaa = NpyAppendArray(targ) - for utt_data in tqdm(data[key], desc=f"dumping {key}"): - npaa.append(utt_data.numpy()) - - assert not os.path.exists(f"{out_prefix}.path.txt") - with open(f"{out_prefix}.path.txt", "w") as f: - for x in data["path"]: - f.write(f"{str(x)}\n") - - assert not os.path.exists(f"{out_prefix}.leng.txt") - with open(f"{out_prefix}.leng.txt", "w") as f: - for x in data["codes"]: - f.write(f"{len(x)}\n") - - assert not os.path.exists(f"{out_prefix}.speaker.txt") - with open(f"{out_prefix}.speaker.txt", "w") as f: - for x in data["speaker"]: - f.write(f"{str(x)}\n") - - print(f"wrote to files with prefix {out_prefix}") - - -def main(meta_path, f0_dir, splits, nshards_list): - speaker_to_stat = defaultdict(F0Stat) - if len(nshards_list) == 1: - nshards_list = nshards_list * len(splits) - else: - assert len(nshards_list) == len(splits) - - for split, nshards in zip(splits, nshards_list): - config, audio_paths, codes, durs, speakers = load_meta(meta_path, split) - path_to_f0 = load_f0(f"{f0_dir}/{split}", nshards) - - # segment-level data - data = prepare_seg_data(config, audio_paths, codes, durs, speakers, path_to_f0) - dump_seg_data(data, config.manifests[split]) - - # speaker f0 - for audio_path, speaker in tqdm(zip(audio_paths, speakers)): - f0 = path_to_f0[audio_path] - speaker_to_stat[speaker].update(f0) - dump_speaker_f0_stat(speaker_to_stat, config.manifests[split]) - - -if __name__ == "__main__": - import argparse - - parser = argparse.ArgumentParser() - parser.add_argument("meta_path") - parser.add_argument("f0_dir", help="out_dir from preprocess_f0") - parser.add_argument("--splits", nargs="+", default=["train", "valid"]) - parser.add_argument( - "--nshards_list", type=int, nargs="+", default=[20], help="number of f0 shards" - ) - args = parser.parse_args() - print(args) - - main(**vars(args)) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/preprocess_f0.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/preprocess_f0.py deleted file mode 100644 index ec86ce063dde3b4606dd9b3202ebfe8a0a020d77..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/preprocess_f0.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. - -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import os - -import torch -from data_utils import load_audio_path -from fairseq.data.codedataset import get_f0_by_filename -from tqdm import tqdm - - -def process_one(path, sr): - """ - Args: - path: audio file path - sr: sampling rate - """ - try: - # YAAPT throws errors in some rare cases - f0 = get_f0_by_filename(path, sr) - except Exception as e: - print( - f"WARNING: error when processing {path}. set f0 to zero. original error message:\n{e}" - ) - f0 = None - return f0 - - -def main(file_path, out_dir, nshards, rank, sampling_rate): - # load data - audio_paths = load_audio_path(file_path) - - # shard - assert nshards <= len(audio_paths) and nshards > 0 - shard_size = len(audio_paths) / nshards - s = int(round((rank - 1) * shard_size)) - e = int(round(rank * shard_size)) - audio_paths = audio_paths[s:e] - - # process - path_to_f0 = {} - for i, audio_path in enumerate(tqdm(audio_paths)): - f0 = process_one(audio_path, sampling_rate) - path_to_f0[audio_path] = f0 - print(f"finished processing {len(path_to_f0)} utterances ({s}-{e})") - - f0_path = f"{out_dir}/f0_{rank}_{nshards}.pt" - os.makedirs(out_dir, exist_ok=True) - torch.save(path_to_f0, f0_path) - print(f"saved to {f0_path}") - - -if __name__ == "__main__": - import argparse - - parser = argparse.ArgumentParser() - parser.add_argument("file_path") - parser.add_argument("out_dir") - parser.add_argument("--nshards", type=int, default=20) - parser.add_argument("--rank", type=int, default=1) - parser.add_argument("--sampling_rate", type=int, default=16000) - args = parser.parse_args() - - main(**vars(args)) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/quantize_f0.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/quantize_f0.py deleted file mode 100644 index 06c6023dbb1503fe764c7fe3b8988735dc013b42..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/quantize_f0.py +++ /dev/null @@ -1,93 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. - -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from collections import defaultdict -from functools import partial - -import numpy as np -import torch -from data_utils import F0Stat, dump_speaker_f0_stat, load_audio_path, load_f0 -from tqdm import tqdm - - -def load_speaker(path): - speakers = [] - with open(path) as f: - for line in f.readlines(): - sample = eval(line.strip()) - assert "speaker" in sample - speakers.append(sample["speaker"]) - return speakers - - -def quantize_f0(speaker_to_f0, f0_stats, nbins, normalize, log): - f0_all = [] - for speaker, f0 in speaker_to_f0.items(): - f0 = f0.raw_data - if log: - f0 = f0.log() - mean = f0_stats[speaker]["logf0_mean"] if log else f0_stats[speaker]["f0_mean"] - std = f0_stats[speaker]["logf0_std"] if log else f0_stats[speaker]["f0_std"] - if normalize == "mean": - f0 = f0 - mean - elif normalize == "meanstd": - f0 = (f0 - mean) / std - f0_all.extend(f0.tolist()) - - hist, bin_x = np.histogram(f0_all, 100000) - cum_hist = np.cumsum(hist) / len(f0_all) * 100 - - f0_bin = {} - for num_bin in nbins: - bin_offset = [] - bin_size = 100 / num_bin - threshold = bin_size - for i in range(num_bin - 1): - index = (np.abs(cum_hist - threshold)).argmin() - bin_offset.append(bin_x[index]) - threshold += bin_size - f0_bin[num_bin] = np.array(bin_offset) - - return f0_bin - - -def main(file_path, f0_dir, out_dir, out_prefix, nbins, nshards, normalize, log): - audio_paths = load_audio_path(file_path) - path_to_f0 = load_f0(f0_dir, nshards) - - speakers = load_speaker(file_path) - speaker_to_f0 = defaultdict(partial(F0Stat, True)) - - # speaker f0 stats - for audio_path, speaker in tqdm(zip(audio_paths, speakers)): - f0 = path_to_f0[audio_path] - speaker_to_f0[speaker].update(f0) - f0_stats = dump_speaker_f0_stat(speaker_to_f0, f"{out_dir}/{out_prefix}") - - # quantize - f0_bin = quantize_f0(speaker_to_f0, f0_stats, nbins, normalize, log) - log_suffix = "_log" if log else "" - f0_bin_out_file = f"{out_dir}/{out_prefix}_{normalize}_norm{log_suffix}_f0_bin.th" - torch.save(f0_bin, f0_bin_out_file) - - -if __name__ == "__main__": - import argparse - - parser = argparse.ArgumentParser() - parser.add_argument("file_path") - parser.add_argument("f0_dir", help="out_dir from preprocess_f0") - parser.add_argument("out_dir") - parser.add_argument("out_prefix") - parser.add_argument("--nbins", nargs="+", type=int, default=[32]) - parser.add_argument("--nshards", type=int, default=20, help="number of f0 shards") - parser.add_argument( - "--normalize", type=str, choices=["meanstd", "mean", "none"], default="mean" - ) - parser.add_argument("--log", action="store_true") - args = parser.parse_args() - print(args) - - main(**vars(args)) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/sample/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/sample/__init__.py deleted file mode 100644 index 0e028c26b9abf533d16fa07d00d942ca07bbba61..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/sample/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. - -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/sample/sample.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/sample/sample.py deleted file mode 100644 index e0f0fd109a75b13df0ebfb601a5d2c8e64b54601..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/sample/sample.py +++ /dev/null @@ -1,609 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import json -import os -import pathlib -import random -import sys - -import numpy as np -import torch -import torch.multiprocessing as mp -import tqdm -from fairseq import checkpoint_utils, options, utils -from fairseq.data.codedataset import CodeDataset, ExpressiveCodeDataConfig -from fairseq.dataclass.utils import convert_namespace_to_omegaconf -from fairseq.utils import move_to_cuda -from torch.distributions.categorical import Categorical -from torch.utils.data import DataLoader, DistributedSampler - -sys.path.append(str(pathlib.Path(__file__).parent.parent)) -from inference_dataset import InferenceDataset, explode_batch -from naive_decoder import Naive_F0_Decoder -from truncated_laplace import truncated_laplace - -CODETYPE_TO_FRAMETIME = {"cpc_km100": 0.01, "hubert": 0.02} # 10ms # 20ms - - -class TemperatureDecoder: - def __init__(self, Ts, discrete_dur=False, discrete_f0=False): - self.T_token, self.T_dur, self.T_f0 = Ts - self.discrete_dur = discrete_dur - self.discrete_f0 = discrete_f0 - - def __call__(self, output): - def sample_multinomial(key, T): - logits = output[key][:, -1, :].float() - return Categorical(logits=logits / T).sample().unsqueeze(-1) - - def sample_laplace(key, T, truncate_at_zero): - mean = output[key][:, -1, :].float() - return truncated_laplace(mean=mean, T=T, truncate_by_zero=truncate_at_zero) - - if self.T_token > 0: - new_tokens = sample_multinomial("token", self.T_token) - else: - new_tokens = output["token"][:, -1, :].argmax(dim=-1, keepdim=True) - - if not self.discrete_dur and self.T_dur == 0: - new_durations = output["duration"][:, -1].round().int() - elif not self.discrete_dur and self.T_dur > 0: - new_durations = ( - sample_laplace("duration", self.T_dur, truncate_at_zero=True) - .round() - .int() - ) - elif self.discrete_dur and self.T_dur > 0: - new_durations = sample_multinomial("duration", self.T_dur) - elif self.discrete_dur and self.T_dur == 0: - new_durations = output["duration"][:, -1, :].argmax(dim=-1, keepdim=True) - else: - assert False - - if not self.discrete_f0 and self.T_f0 == 0: - new_f0 = output["f0"][:, -1] - elif not self.discrete_f0 and self.T_f0 > 0: - new_f0 = sample_laplace("f0", self.T_f0, truncate_at_zero=False) - elif self.discrete_f0 and self.T_f0 > 0: - new_f0 = sample_multinomial("f0", self.T_f0) - elif self.discrete_f0 and self.T_f0 == 0: - new_f0 = output["f0"][:, -1, :].argmax(dim=-1, keepdim=True) - else: - assert False - - return new_tokens, new_durations, new_f0 - - -class FilterNamesDataset: - def __init__(self, dataset, fnames_path): - self.dataset = dataset - - with open(fnames_path, "r") as fin: - fnames = set((eval(line)["audio"] for line in fin)) - print(f"# will retrict the dataset for {len(fnames)} files") - - self.indexes = [] - - for i, datapoint in enumerate(dataset): - if datapoint["filename"] in fnames: - self.indexes.append(i) - assert len(self.indexes) == len(fnames), f"{len(self.indexes)} {len(fnames)}" - - self.collater = self.dataset.collater - self.discrete_dur = self.dataset.discrete_dur - self.discrete_f0 = self.dataset.discrete_f0 - - def __len__(self): - return len(self.indexes) - - def __getitem__(self, k): - k = self.indexes[k] - return self.dataset[k] - - def size(self, k): - k = self.indexes[k] - return self.dataset.size(k) - - -@torch.no_grad() -def do_sampling( - model, - batch, - eos_token, - decoder, - autoregressive_steps=100, - teacher_force_tokens=False, - teacher_force_duration=False, - teacher_force_f0=False, - match_duration=False, -): - def autoregressive_step_(output, autoregressive_steps): - new_tokens, new_durations, new_f0 = decoder(output) - - n = output["token"].size(1) if output["token"].ndim == 3 else 1 - - if teacher_force_tokens: - new_tokens = batch["target"][:, n - 1].unsqueeze(-1) - if teacher_force_duration: - new_durations = batch["dur_target"][:, n - 1].unsqueeze(-1) - if teacher_force_f0: - new_f0 = batch["f0_target"][:, n - 1].unsqueeze(-1) - - batch["net_input"]["src_tokens"] = torch.cat( - [batch["net_input"]["src_tokens"], new_tokens], dim=1 - ) - batch["net_input"]["dur_src"] = torch.cat( - [batch["net_input"]["dur_src"], new_durations], dim=1 - ) - batch["net_input"]["f0_src"] = torch.cat( - [batch["net_input"]["f0_src"], new_f0], dim=1 - ) - - outputs = [] - - if teacher_force_tokens or teacher_force_duration or teacher_force_f0: - max_time = batch["target"].size(1) - prefix_time = batch["net_input"]["src_tokens"].size(1) - - autoregressive_steps = max_time - prefix_time + 1 # should be 0 - - for _ in range(autoregressive_steps): - output = model(**batch["net_input"]) - - last_steps = ( - output["token"][:, -1, ...], - output["duration"][:, -1, ...], - output["f0"][:, -1, ...], - ) - outputs.append(last_steps) - - autoregressive_step_(output, autoregressive_steps) - tokens, duration, f0 = ( - batch["net_input"]["src_tokens"], - batch["net_input"]["dur_src"], - batch["net_input"]["f0_src"], - ) - - if ( - match_duration - and (batch["dur_target"].sum(dim=-1) < duration.sum(dim=-1)).all() - ): - break - - return tokens, duration, f0, outputs - - -def unroll_duration(token_stream, duration_stream): - assert len(token_stream) == len( - duration_stream - ), f"{len(token_stream)} != {len(duration_stream)}" - non_positive_durations = sum(d <= 0 for d in duration_stream) - if non_positive_durations > 0: - print( - f"# {non_positive_durations} durations are non-positive, they will be capped to 1" - ) - - result = [] - - duration_stream_rounded_capped = [max(1, int(round(x))) for x in duration_stream] - for t, d in zip(token_stream, duration_stream_rounded_capped): - result.extend([t] * d) - - return result - - -def realign_shifted_streams(tokens, durations, F0s, shifts): - """ - Durations are shifted by 1, F0 by 2 - >>> tokens = ["", "t1", "t2", "t3", "", "x", "x"] - >>> durations = ["<0>", "<0>", "d1", "d2", "d3", "<0>", "x"] - >>> F0s = ["<0>", "<0>", "<0>", "f1", "f2", "f3", "<0>"] - >>> shifts = [1,2] - >>> realign_shifted_streams(tokens, durations, F0s, shifts) - (['', 't1', 't2', 't3', ''], ['<0>', 'd1', 'd2', 'd3', '<0>'], ['<0>', 'f1', 'f2', 'f3', '<0>']) - """ - max_shift = max(shifts) - if max_shift > 0: - shift_durations, shift_F0s = shifts - - tokens = tokens[:-max_shift] - durations = durations[shift_durations:] - if shift_durations < max_shift: - durations = durations[: -(max_shift - shift_durations)] - - if F0s is not None: - F0s = F0s[shift_F0s:] - if shift_F0s < max_shift: - F0s = F0s[: -(max_shift - shift_F0s)] - - assert len(tokens) == len(durations), f"{len(tokens)} =! {len(durations)}" - if F0s is not None: - assert len(tokens) == len(F0s), f"{len(tokens)} =! {len(F0s)}" - - return tokens, durations, F0s - - -def maybe_cut_eos(produced_tokens, produced_duration, produced_f0, eos_idx): - if eos_idx in produced_tokens: - eos_index = produced_tokens.index(eos_idx) - produced_tokens = produced_tokens[:eos_index] - produced_duration = produced_duration[:eos_index] - produced_f0 = produced_f0[:eos_index] - return produced_tokens, produced_duration, produced_f0 - - -def maybe_filter_pad(produced_tokens, produced_duration, produced_f0, pad_idx): - if pad_idx not in produced_tokens: - return produced_tokens, produced_duration, produced_f0 - - assert len(produced_tokens) == len(produced_duration) == len(produced_f0) - - print(" is detected in the output!") - filtered_tokens, filtered_duration, filtered_f0 = [], [], [] - - for t, d, f in zip(produced_tokens, produced_duration, produced_f0): - if t != pad_idx: - filtered_tokens.append(t) - filtered_duration.append(d) - filtered_f0.append(f) - return filtered_tokens, filtered_duration, filtered_f0 - - -def match_duration(produced_tokens, produced_duration, produced_f0, target_duration): - """ - >>> tokens = ['t'] * 4 - >>> F0s = ['f0'] * 4 - >>> produced_duration = [1, 10, 10, 10] - >>> match_duration(tokens, produced_duration, F0s, target_duration=100) - (['t', 't', 't', 't'], [1, 10, 10, 10], ['f0', 'f0', 'f0', 'f0']) - >>> match_duration(tokens, produced_duration, F0s, target_duration=5) - (['t', 't'], [1, 4], ['f0', 'f0']) - """ - if sum(produced_duration) <= target_duration: - return produced_tokens, produced_duration, produced_f0 - - running_duration = 0 - filtered_duration = [] - - for next_tok_duration in produced_duration: - if running_duration + next_tok_duration < target_duration: - filtered_duration.append(next_tok_duration) - running_duration += next_tok_duration - else: - to_add = target_duration - running_duration - assert to_add <= next_tok_duration - filtered_duration.append(to_add) - break - - produced_duration = filtered_duration - assert sum(produced_duration) == target_duration - - n_tok = len(filtered_duration) - - return produced_tokens[:n_tok], produced_duration, produced_f0[:n_tok] - - -def main(rank, world_size, args): - if world_size > 1: - torch.distributed.init_process_group( - backend="gloo", init_method="env://", world_size=world_size, rank=rank - ) - torch.cuda.set_device(rank) - - raw_args = args - args = convert_namespace_to_omegaconf(args) - if args.common.seed is not None: - random.seed(args.common.seed) - np.random.seed(args.common.seed) - utils.set_torch_seed(args.common.seed) - - models, model_args, task = checkpoint_utils.load_model_ensemble_and_task( - [raw_args.path], arg_overrides={"data": args.task.data} - ) - tgt_dict = task.target_dictionary - - for model in models: - model.prepare_for_inference_(args) - model.cuda().eval() - if raw_args.fp16: - model = model.half() - model = models[0] - - config = ExpressiveCodeDataConfig(args.task.data) - - dataset = CodeDataset( - manifest=config.manifests[raw_args.subset], - dictionary=task.source_dictionary, - dur_dictionary=task.source_duration_dictionary, - f0_dictionary=task.source_f0_dictionary, - config=config, - discrete_dur=task.cfg.discrete_duration, - discrete_f0=task.cfg.discrete_f0, - log_f0=task.cfg.log_f0, - normalize_f0_mean=task.cfg.normalize_f0_mean, - normalize_f0_std=task.cfg.normalize_f0_std, - interpolate_f0=task.cfg.interpolate_f0, - shifts=task.cfg.stream_shifts, - return_filename=True, - strip_filename=False, - ) - tgt_dict = task.target_dictionary - shifts = dataset.shifts.dur, dataset.shifts.f0 - max_shift = max(shifts) - - fname = raw_args.output - if world_size > 1: - fname += f"_{rank}" - output_file = open(fname, "w") - - if raw_args.filter_names: - dataset = FilterNamesDataset(dataset, raw_args.filter_names) - - dataset = InferenceDataset(dataset, raw_args.prefix_length, filter_short=True) - print(f"Dataset size {len(dataset)}") - sampler = ( - None - if world_size == 1 - else DistributedSampler( - dataset, num_replicas=world_size, rank=rank, shuffle=False - ) - ) - dataloader = DataLoader( - dataset, - batch_size=1, - shuffle=False, - collate_fn=dataset.collater, - sampler=sampler, - ) - - Ts = raw_args.T_token, raw_args.T_duration, raw_args.T_f0 - decoder = TemperatureDecoder( - Ts, discrete_dur=task.cfg.discrete_duration, discrete_f0=task.cfg.discrete_f0 - ) - - dataset_size = len(dataset) - - f0_decoder = None - if raw_args.f0_discretization_bounds: - assert task.cfg.discrete_f0 - f0_decoder = Naive_F0_Decoder(raw_args.f0_discretization_bounds).cuda() - - pbar = ( - tqdm.tqdm( - total=dataset_size - if raw_args.max_samples is None - else min(raw_args.max_samples, dataset_size) - ) - if world_size == 1 - else None - ) - - samples_produced = 0 - - for batch in dataloader: - if ( - raw_args.max_samples is not None - and samples_produced >= raw_args.max_samples - ): - break - - prefix = batch["prefix"][0] - - batch = explode_batch(batch, raw_args.batch_explosion_rate) - batch = move_to_cuda(batch) - - if not raw_args.short_curcuit: - produced_tokens, produced_durations, produced_f0, _ = do_sampling( - models[0], - batch, - tgt_dict.eos(), - decoder, - autoregressive_steps=raw_args.max_length - prefix + max_shift, - teacher_force_tokens=raw_args.teacher_force_tokens, - match_duration=raw_args.match_duration, - teacher_force_duration=raw_args.teacher_force_duration, - teacher_force_f0=raw_args.teacher_force_f0, - ) - - # stip entries corresponding to - produced_tokens = produced_tokens[:, 1:] - produced_durations = produced_durations[:, 1:] - produced_f0 = produced_f0[:, 1:] - - else: - max_length = raw_args.max_length + max_shift - produced_tokens, produced_durations, produced_f0 = ( - batch["target"][:, :max_length], - batch["dur_target"][:, :max_length], - batch["f0_target"][:, :max_length], - ) - - if f0_decoder is not None: - produced_f0 = f0_decoder(produced_f0) - - produced_tokens, produced_durations, produced_f0 = ( - produced_tokens.cpu().tolist(), - produced_durations.cpu().tolist(), - produced_f0.cpu().tolist(), - ) - - bsz = batch["target"].size(0) - assert bsz == raw_args.batch_explosion_rate - - for i in range(bsz): - if ( - raw_args.max_samples is not None - and samples_produced >= raw_args.max_samples - ): - break - - produced_tokens_i = produced_tokens[i] - produced_durations_i = produced_durations[i] - produced_f0_i = produced_f0[i] - - ( - produced_tokens_i, - produced_durations_i, - produced_f0_i, - ) = realign_shifted_streams( - produced_tokens_i, produced_durations_i, produced_f0_i, shifts - ) - - produced_tokens_i, produced_durations_i, produced_f0_i = maybe_cut_eos( - produced_tokens_i, produced_durations_i, produced_f0_i, tgt_dict.eos() - ) - - produced_tokens_i, produced_durations_i, produced_f0_i = maybe_filter_pad( - produced_tokens_i, produced_durations_i, produced_f0_i, tgt_dict.pad() - ) - - if raw_args.match_duration: - # NB: here we cheat a bit and use that padding has duration 0 - # so no need to re-align and remove padding - dur_target_i = batch["dur_target"][i, :].sum().item() - produced_tokens_i, produced_durations_i, produced_f0_i = match_duration( - produced_tokens_i, produced_durations_i, produced_f0_i, dur_target_i - ) - - if raw_args.cut_prompt: - produced_tokens_i, produced_durations_i, produced_f0_i = ( - produced_tokens_i[prefix:], - produced_durations_i[prefix:], - produced_f0_i[prefix:], - ) - - prompt_fname = batch["filename"][0] - fname = str(pathlib.Path(prompt_fname).with_suffix("")) + f"__{i}.wav" - - token_stream = unroll_duration(produced_tokens_i, produced_durations_i) - f0_stream = unroll_duration(produced_f0_i, produced_durations_i) - output_line = json.dumps( - { - "audio": fname, - "prompt": prompt_fname, - raw_args.code_type: " ".join(map(str, token_stream)), - "duration": round( - sum(produced_durations_i) - * CODETYPE_TO_FRAMETIME[raw_args.code_type], - 3, - ), - "raw_duration": produced_durations_i, - "raw_f0": produced_f0_i, - "f0": [round(f0, 3) for f0 in f0_stream], - } - ) - print(output_line, file=output_file) - - if pbar: - pbar.update(1) - samples_produced += 1 - - if raw_args.debug: - break - - output_file.close() - - if world_size > 1: - # important that everything is flushed before aggregating - torch.distributed.barrier() - - if world_size > 1 and rank == 0: - with open(raw_args.output, "w") as fout: - for i in range(world_size): - f = raw_args.output + f"_{i}" - with open(f, "r") as fin: - fout.write(fin.read()) - os.remove(f) - - -def cli_main(): - parser = options.get_interactive_generation_parser() - parser.add_argument( - "--prefix-length", - type=int, - default=1, - help="Prompt prefix length (including )", - ) - parser.add_argument("--output", type=str, default=None, required=True) - parser.add_argument( - "--debug", action="store_true", help="Process only the first batch" - ) - parser.add_argument( - "--ignore-durations", - action="store_true", - help="If set, the duration stream is ignored", - ) - parser.add_argument( - "--max-length", type=int, default=200, help="Maximal produced length" - ) - parser.add_argument( - "--code-type", choices=["cpc_km100", "hubert"], default="cpc_km100" - ) - parser.add_argument("--max-samples", type=int, default=None) - parser.add_argument("--prompt-duration-scaler", type=float, default=1.0) - parser.add_argument("--teacher-force-tokens", action="store_true", default=False) - parser.add_argument("--teacher-force-duration", action="store_true", default=False) - parser.add_argument("--teacher-force-f0", action="store_true", default=False) - parser.add_argument("--filter-names", type=str, default=None) - parser.add_argument( - "--match-duration", - action="store_true", - help="Do not produce sequences longer that ground-truth", - ) - parser.add_argument( - "--cut-prompt", - action="store_true", - help="Remove prompt from the produced audio", - ) - parser.add_argument( - "--short-curcuit", action="store_true", help="Use 'target' as a sample" - ) - parser.add_argument("--f0-discretization-bounds", type=str, default=None) - - parser.add_argument("--batch-explosion-rate", type=int, default=1) - - parser.add_argument("--T-token", type=float, default=1.0) - parser.add_argument("--T-duration", type=float, default=1.0) - parser.add_argument("--T-f0", type=float, default=1.0) - - parser.add_argument( - "--subset", type=str, default="valid", choices=["test", "valid"] - ) - - args = options.parse_args_and_arch(parser) - - assert ( - args.prefix_length >= 1 - ), "Prefix length includes bos token , hence the minimum is 1." - assert all( - t >= 0 for t in [args.T_token, args.T_f0, args.T_duration] - ), "T must be non-negative!" - - world_size = torch.cuda.device_count() - if world_size > 1: - import random - - mp.set_start_method("spawn", force=True) - os.environ["MASTER_ADDR"] = "localhost" - os.environ["MASTER_PORT"] = str(random.randint(10_000, 50_000)) - - print(f"Using {world_size} devices, master port {os.environ['MASTER_PORT']}") - - mp.spawn( - main, - nprocs=world_size, - args=( - world_size, - args, - ), - join=True, - ) - else: - main(rank=0, world_size=world_size, args=args) - - -if __name__ == "__main__": - cli_main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/scripts/join_units_manifest.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/scripts/join_units_manifest.py deleted file mode 100644 index b2de2b9d1db5d39f6cc71e21bd7bd9395951b627..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/scripts/join_units_manifest.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. - -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import json -import pathlib - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--manifest", required=True) - parser.add_argument("--units", required=True) - parser.add_argument("--output", required=True) - parser.add_argument("--sample_rate", type=int, default=16_000) - - args = parser.parse_args() - - with open(args.manifest, "r") as manifest, open(args.units, "r") as units, open( - args.output, "w" - ) as outp: - root = manifest.readline().strip() - root = pathlib.Path(root) - - for manifest_line, unit_line in zip(manifest.readlines(), units.readlines()): - path, frames = manifest_line.split() - duration = int(frames) / float(args.sample_rate) - fname = root / path - speaker = fname.parent.parent.name - - units = unit_line.split("|")[1] - - print( - json.dumps( - dict( - audio=str(root / path), - duration=duration, - hubert_km100=units.strip(), - speaker=speaker, - ) - ), - file=outp, - ) - - -if __name__ == "__main__": - main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/scripts/prepare_data.sh b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/scripts/prepare_data.sh deleted file mode 100644 index ec892e59a4c6aa416e7871e89f443cd046294980..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/scripts/prepare_data.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash -# Copyright (c) Facebook, Inc. and its affiliates. - -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -set -eu - -train_json=$1 -valid_json=$2 -test_json=$3 -n_units=$4 -hop_size=$5 -sr=$6 -f0_quantizer=$7 -out_dir=$8 - -meta_path="$out_dir/data_config.json" -f0_dir="$out_dir/f0" - -mkdir -p $out_dir -ln -sf $train_json $out_dir/train.txt -ln -sf $valid_json $out_dir/valid.txt -ln -sf $test_json $out_dir/test.txt - -cat <$meta_path -{ - "manifests": { - "train": "$out_dir/train.txt", - "valid": "$out_dir/valid.txt", - "test": "$out_dir/test.txt" - }, - "n_units": $n_units, - "code_hop_size": $hop_size, - "sampling_rate": $sr, - "multispkr": "parent_parent_name", - - "f0_vq_type": "naive", - "f0_vq_naive_quantizer": { - "log_mean_norm": "$f0_quantizer" - }, - "f0_vq_n_units": 32 -} -EOF - -for split in train valid test; do - python examples/textless_nlp/pgslm/preprocess_f0.py \ - $out_dir/$split.txt $f0_dir/$split --nshards=1 --rank=1 --sampling_rate=$sr - - #NSHARDS=16 - #seq 1 $NSHARDS | parallel -j $NSHARDS python examples/textless_nlp/pgslm/preprocess_f0.py \ - # $out_dir/$split.txt $f0_dir/$split --nshards=$NSHARDS --sampling_rate=$sr --rank -done - -# Please make sure that the number of shards (--nshards_list) is consistent across commands -python examples/textless_nlp/pgslm/prepare_dataset.py \ - $meta_path $f0_dir --splits test valid train --nshards_list 1 diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/scripts/prepare_f0_quantization.sh b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/scripts/prepare_f0_quantization.sh deleted file mode 100644 index 3a285a39bc2bea3240b0f0e38ee71fa872fcadf5..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/scripts/prepare_f0_quantization.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -# Copyright (c) Facebook, Inc. and its affiliates. - -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -set -eu - -train_json=$1 -sr=$2 -nbins=$3 -out_dir=$4 -out_prefix=$5 - -f0_dir="$out_dir/f0" - -python examples/textless_nlp/pgslm/preprocess_f0.py \ - $train_json $f0_dir/${out_prefix}_f0_quant --nshards 1 --rank 1 --sampling_rate $sr - -# NB: one can use parallel here: -# NSHARDS=16 -# -#seq 1 $NSHARDS | parallel -j $NSHARDS python examples/textless_nlp/pgslm/preprocess_f0.py \ -# $train_json $f0_dir/${out_prefix}_f0_quant --nshards $NSHARDS --sampling_rate $sr --rank - -python examples/textless_nlp/pgslm/quantize_f0.py \ - $train_json $f0_dir/${out_prefix}_f0_quant $out_dir $out_prefix --nbins $nbins --nshards 1 --normalize mean --log diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/truncated_laplace.py b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/truncated_laplace.py deleted file mode 100644 index db2b9cc1f82085af6a034dcc8154a91c81a57ae8..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/pgslm/truncated_laplace.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. - -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import warnings - -import torch - - -def truncated_laplace(mean, T, truncate_by_zero=False): - """Generating a sample from a Laplace distribution, possible left-truncated at zero. - A bit of explanation here https://stats.stackexchange.com/a/357598 . - """ - assert isinstance(mean, torch.Tensor) - - if not truncate_by_zero: - percentile = 0.0 - else: - if not (mean >= 0.0).all(): - warnings.warn(f"means are supposed to be non-negative, but got {mean}") - mean = torch.clamp_min(mean, 0.0) - - lower_bound = mean.new_tensor([0.0]) - percentile = 0.5 + 0.5 * torch.sign(lower_bound - mean) * ( - 1.0 - torch.exp(-1.0 / T * torch.abs(mean - lower_bound)) - ) - - p = torch.empty_like(mean).uniform_() * (1.0 - percentile) + percentile - return mean - T * torch.sign(p - 0.5) * torch.log(1 - 2 * torch.abs(p - 0.5)) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/speech-resynth/README.md b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/speech-resynth/README.md deleted file mode 100644 index a099682cdbdb991e2c37f1a00387e0b8fc8f121c..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/speech-resynth/README.md +++ /dev/null @@ -1,28 +0,0 @@ - -# Speech Resynthesis from Discrete Disentangled Self-Supervised Representations -Landing page with usfull resources for the [Speech Resynthesis from Discrete Disentangled Self-Supervised Representations](https://arxiv.org/abs/2104.00355) paper. - -

- -__Abstract__: We propose using self-supervised discrete representations for the task of speech resynthesis. To generate disentangled representation, we separately extract low-bitrate representations for speech content, prosodic information, and speaker identity. This allows to synthesize speech in a controllable manner. We analyze various state-of-the-art, self-supervised representation learning methods and shed light on the advantages of each method while considering reconstruction quality and disentanglement properties. Specifically, we evaluate the F0 reconstruction, speaker identification performance (for both resynthesis and voice conversion), recordings' intelligibility, and overall quality using subjective human evaluation. Lastly, we demonstrate how these representations can be used for an ultra-lightweight speech codec. Using the obtained representations, we can get to a rate of 365 bits per second while providing better speech quality than the baseline methods. - - -## Quick Links -- [Paper](https://arxiv.org/pdf/2104.00355.pdf) -- [Samples](https://speechbot.github.io/resynthesis/index.html) -- [Code](https://github.com/facebookresearch/speech-resynthesis) - -The codebase for the [Speech Resynthesis from Discrete Disentangled Self-Supervised Representations](https://arxiv.org/abs/2104.00355) paper can be found under the following [repository](https://github.com/facebookresearch/speech-resynthesis). - - -## Citation -``` -@inproceedings{polyak21_interspeech, - author={Adam Polyak and Yossi Adi and Jade Copet and - Eugene Kharitonov and Kushal Lakhotia and - Wei-Ning Hsu and Abdelrahman Mohamed and Emmanuel Dupoux}, - title={{Speech Resynthesis from Discrete Disentangled Self-Supervised Representations}}, - year=2021, - booktitle={Proc. Interspeech 2021}, -} -``` diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/speech-resynth/img/fig.png b/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/speech-resynth/img/fig.png deleted file mode 100644 index f92509bcfef84a7f5ead19f3b224ff89f9ee0a30..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/speech-resynth/img/fig.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c19c570f3671d88551f5d5b908e270e69bd75d304c5d358868fa19f342979c17 -size 307833 diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/translation/README.md b/audio/safeear/safeear_repo/fairseq_ours/examples/translation/README.md deleted file mode 100644 index 2941f5eb8482dab61dca5eca27a71abd7ee5bf5c..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/translation/README.md +++ /dev/null @@ -1,301 +0,0 @@ -# Neural Machine Translation - -This README contains instructions for [using pretrained translation models](#example-usage-torchhub) -as well as [training new models](#training-a-new-model). - -## Pre-trained models - -Model | Description | Dataset | Download ----|---|---|--- -`conv.wmt14.en-fr` | Convolutional
([Gehring et al., 2017](https://arxiv.org/abs/1705.03122)) | [WMT14 English-French](http://statmt.org/wmt14/translation-task.html#Download) | model:
[download (.tar.bz2)](https://dl.fbaipublicfiles.com/fairseq/models/wmt14.v2.en-fr.fconv-py.tar.bz2)
newstest2014:
[download (.tar.bz2)](https://dl.fbaipublicfiles.com/fairseq/data/wmt14.v2.en-fr.newstest2014.tar.bz2)
newstest2012/2013:
[download (.tar.bz2)](https://dl.fbaipublicfiles.com/fairseq/data/wmt14.v2.en-fr.ntst1213.tar.bz2) -`conv.wmt14.en-de` | Convolutional
([Gehring et al., 2017](https://arxiv.org/abs/1705.03122)) | [WMT14 English-German](http://statmt.org/wmt14/translation-task.html#Download) | model:
[download (.tar.bz2)](https://dl.fbaipublicfiles.com/fairseq/models/wmt14.en-de.fconv-py.tar.bz2)
newstest2014:
[download (.tar.bz2)](https://dl.fbaipublicfiles.com/fairseq/data/wmt14.en-de.newstest2014.tar.bz2) -`conv.wmt17.en-de` | Convolutional
([Gehring et al., 2017](https://arxiv.org/abs/1705.03122)) | [WMT17 English-German](http://statmt.org/wmt17/translation-task.html#Download) | model:
[download (.tar.bz2)](https://dl.fbaipublicfiles.com/fairseq/models/wmt17.v2.en-de.fconv-py.tar.bz2)
newstest2014:
[download (.tar.bz2)](https://dl.fbaipublicfiles.com/fairseq/data/wmt17.v2.en-de.newstest2014.tar.bz2) -`transformer.wmt14.en-fr` | Transformer
([Ott et al., 2018](https://arxiv.org/abs/1806.00187)) | [WMT14 English-French](http://statmt.org/wmt14/translation-task.html#Download) | model:
[download (.tar.bz2)](https://dl.fbaipublicfiles.com/fairseq/models/wmt14.en-fr.joined-dict.transformer.tar.bz2)
newstest2014:
[download (.tar.bz2)](https://dl.fbaipublicfiles.com/fairseq/data/wmt14.en-fr.joined-dict.newstest2014.tar.bz2) -`transformer.wmt16.en-de` | Transformer
([Ott et al., 2018](https://arxiv.org/abs/1806.00187)) | [WMT16 English-German](https://drive.google.com/uc?export=download&id=0B_bZck-ksdkpM25jRUN2X2UxMm8) | model:
[download (.tar.bz2)](https://dl.fbaipublicfiles.com/fairseq/models/wmt16.en-de.joined-dict.transformer.tar.bz2)
newstest2014:
[download (.tar.bz2)](https://dl.fbaipublicfiles.com/fairseq/data/wmt16.en-de.joined-dict.newstest2014.tar.bz2) -`transformer.wmt18.en-de` | Transformer
([Edunov et al., 2018](https://arxiv.org/abs/1808.09381))
WMT'18 winner | [WMT'18 English-German](http://www.statmt.org/wmt18/translation-task.html) | model:
[download (.tar.gz)](https://dl.fbaipublicfiles.com/fairseq/models/wmt18.en-de.ensemble.tar.gz)
See NOTE in the archive -`transformer.wmt19.en-de` | Transformer
([Ng et al., 2019](https://arxiv.org/abs/1907.06616))
WMT'19 winner | [WMT'19 English-German](http://www.statmt.org/wmt19/translation-task.html) | model:
[download (.tar.gz)](https://dl.fbaipublicfiles.com/fairseq/models/wmt19.en-de.joined-dict.ensemble.tar.gz) -`transformer.wmt19.de-en` | Transformer
([Ng et al., 2019](https://arxiv.org/abs/1907.06616))
WMT'19 winner | [WMT'19 German-English](http://www.statmt.org/wmt19/translation-task.html) | model:
[download (.tar.gz)](https://dl.fbaipublicfiles.com/fairseq/models/wmt19.de-en.joined-dict.ensemble.tar.gz) -`transformer.wmt19.en-ru` | Transformer
([Ng et al., 2019](https://arxiv.org/abs/1907.06616))
WMT'19 winner | [WMT'19 English-Russian](http://www.statmt.org/wmt19/translation-task.html) | model:
[download (.tar.gz)](https://dl.fbaipublicfiles.com/fairseq/models/wmt19.en-ru.ensemble.tar.gz) -`transformer.wmt19.ru-en` | Transformer
([Ng et al., 2019](https://arxiv.org/abs/1907.06616))
WMT'19 winner | [WMT'19 Russian-English](http://www.statmt.org/wmt19/translation-task.html) | model:
[download (.tar.gz)](https://dl.fbaipublicfiles.com/fairseq/models/wmt19.ru-en.ensemble.tar.gz) - -## Example usage (torch.hub) - -We require a few additional Python dependencies for preprocessing: -```bash -pip install fastBPE sacremoses subword_nmt -``` - -Interactive translation via PyTorch Hub: -```python -import torch - -# List available models -torch.hub.list('pytorch/fairseq') # [..., 'transformer.wmt16.en-de', ... ] - -# Load a transformer trained on WMT'16 En-De -# Note: WMT'19 models use fastBPE instead of subword_nmt, see instructions below -en2de = torch.hub.load('pytorch/fairseq', 'transformer.wmt16.en-de', - tokenizer='moses', bpe='subword_nmt') -en2de.eval() # disable dropout - -# The underlying model is available under the *models* attribute -assert isinstance(en2de.models[0], fairseq.models.transformer.TransformerModel) - -# Move model to GPU for faster translation -en2de.cuda() - -# Translate a sentence -en2de.translate('Hello world!') -# 'Hallo Welt!' - -# Batched translation -en2de.translate(['Hello world!', 'The cat sat on the mat.']) -# ['Hallo Welt!', 'Die Katze saß auf der Matte.'] -``` - -Loading custom models: -```python -from fairseq.models.transformer import TransformerModel -zh2en = TransformerModel.from_pretrained( - '/path/to/checkpoints', - checkpoint_file='checkpoint_best.pt', - data_name_or_path='data-bin/wmt17_zh_en_full', - bpe='subword_nmt', - bpe_codes='data-bin/wmt17_zh_en_full/zh.code' -) -zh2en.translate('你好 世界') -# 'Hello World' -``` - -If you are using a `transformer.wmt19` models, you will need to set the `bpe` -argument to `'fastbpe'` and (optionally) load the 4-model ensemble: -```python -en2de = torch.hub.load('pytorch/fairseq', 'transformer.wmt19.en-de', - checkpoint_file='model1.pt:model2.pt:model3.pt:model4.pt', - tokenizer='moses', bpe='fastbpe') -en2de.eval() # disable dropout -``` - -## Example usage (CLI tools) - -Generation with the binarized test sets can be run in batch mode as follows, e.g. for WMT 2014 English-French on a GTX-1080ti: -```bash -mkdir -p data-bin -curl https://dl.fbaipublicfiles.com/fairseq/models/wmt14.v2.en-fr.fconv-py.tar.bz2 | tar xvjf - -C data-bin -curl https://dl.fbaipublicfiles.com/fairseq/data/wmt14.v2.en-fr.newstest2014.tar.bz2 | tar xvjf - -C data-bin -fairseq-generate data-bin/wmt14.en-fr.newstest2014 \ - --path data-bin/wmt14.en-fr.fconv-py/model.pt \ - --beam 5 --batch-size 128 --remove-bpe | tee /tmp/gen.out -# ... -# | Translated 3003 sentences (96311 tokens) in 166.0s (580.04 tokens/s) -# | Generate test with beam=5: BLEU4 = 40.83, 67.5/46.9/34.4/25.5 (BP=1.000, ratio=1.006, syslen=83262, reflen=82787) - -# Compute BLEU score -grep ^H /tmp/gen.out | cut -f3- > /tmp/gen.out.sys -grep ^T /tmp/gen.out | cut -f2- > /tmp/gen.out.ref -fairseq-score --sys /tmp/gen.out.sys --ref /tmp/gen.out.ref -# BLEU4 = 40.83, 67.5/46.9/34.4/25.5 (BP=1.000, ratio=1.006, syslen=83262, reflen=82787) -``` - -## Training a new model - -### IWSLT'14 German to English (Transformer) - -The following instructions can be used to train a Transformer model on the [IWSLT'14 German to English dataset](http://workshop2014.iwslt.org/downloads/proceeding.pdf). - -First download and preprocess the data: -```bash -# Download and prepare the data -cd examples/translation/ -bash prepare-iwslt14.sh -cd ../.. - -# Preprocess/binarize the data -TEXT=examples/translation/iwslt14.tokenized.de-en -fairseq-preprocess --source-lang de --target-lang en \ - --trainpref $TEXT/train --validpref $TEXT/valid --testpref $TEXT/test \ - --destdir data-bin/iwslt14.tokenized.de-en \ - --workers 20 -``` - -Next we'll train a Transformer translation model over this data: -```bash -CUDA_VISIBLE_DEVICES=0 fairseq-train \ - data-bin/iwslt14.tokenized.de-en \ - --arch transformer_iwslt_de_en --share-decoder-input-output-embed \ - --optimizer adam --adam-betas '(0.9, 0.98)' --clip-norm 0.0 \ - --lr 5e-4 --lr-scheduler inverse_sqrt --warmup-updates 4000 \ - --dropout 0.3 --weight-decay 0.0001 \ - --criterion label_smoothed_cross_entropy --label-smoothing 0.1 \ - --max-tokens 4096 \ - --eval-bleu \ - --eval-bleu-args '{"beam": 5, "max_len_a": 1.2, "max_len_b": 10}' \ - --eval-bleu-detok moses \ - --eval-bleu-remove-bpe \ - --eval-bleu-print-samples \ - --best-checkpoint-metric bleu --maximize-best-checkpoint-metric -``` - -Finally we can evaluate our trained model: -```bash -fairseq-generate data-bin/iwslt14.tokenized.de-en \ - --path checkpoints/checkpoint_best.pt \ - --batch-size 128 --beam 5 --remove-bpe -``` - -### WMT'14 English to German (Convolutional) - -The following instructions can be used to train a Convolutional translation model on the WMT English to German dataset. -See the [Scaling NMT README](../scaling_nmt/README.md) for instructions to train a Transformer translation model on this data. - -The WMT English to German dataset can be preprocessed using the `prepare-wmt14en2de.sh` script. -By default it will produce a dataset that was modeled after [Attention Is All You Need (Vaswani et al., 2017)](https://arxiv.org/abs/1706.03762), but with additional news-commentary-v12 data from WMT'17. - -To use only data available in WMT'14 or to replicate results obtained in the original [Convolutional Sequence to Sequence Learning (Gehring et al., 2017)](https://arxiv.org/abs/1705.03122) paper, please use the `--icml17` option. - -```bash -# Download and prepare the data -cd examples/translation/ -# WMT'17 data: -bash prepare-wmt14en2de.sh -# or to use WMT'14 data: -# bash prepare-wmt14en2de.sh --icml17 -cd ../.. - -# Binarize the dataset -TEXT=examples/translation/wmt17_en_de -fairseq-preprocess \ - --source-lang en --target-lang de \ - --trainpref $TEXT/train --validpref $TEXT/valid --testpref $TEXT/test \ - --destdir data-bin/wmt17_en_de --thresholdtgt 0 --thresholdsrc 0 \ - --workers 20 - -# Train the model -mkdir -p checkpoints/fconv_wmt_en_de -fairseq-train \ - data-bin/wmt17_en_de \ - --arch fconv_wmt_en_de \ - --dropout 0.2 \ - --criterion label_smoothed_cross_entropy --label-smoothing 0.1 \ - --optimizer nag --clip-norm 0.1 \ - --lr 0.5 --lr-scheduler fixed --force-anneal 50 \ - --max-tokens 4000 \ - --save-dir checkpoints/fconv_wmt_en_de - -# Evaluate -fairseq-generate data-bin/wmt17_en_de \ - --path checkpoints/fconv_wmt_en_de/checkpoint_best.pt \ - --beam 5 --remove-bpe -``` - -### WMT'14 English to French -```bash -# Download and prepare the data -cd examples/translation/ -bash prepare-wmt14en2fr.sh -cd ../.. - -# Binarize the dataset -TEXT=examples/translation/wmt14_en_fr -fairseq-preprocess \ - --source-lang en --target-lang fr \ - --trainpref $TEXT/train --validpref $TEXT/valid --testpref $TEXT/test \ - --destdir data-bin/wmt14_en_fr --thresholdtgt 0 --thresholdsrc 0 \ - --workers 60 - -# Train the model -mkdir -p checkpoints/fconv_wmt_en_fr -fairseq-train \ - data-bin/wmt14_en_fr \ - --arch fconv_wmt_en_fr \ - --dropout 0.1 \ - --criterion label_smoothed_cross_entropy --label-smoothing 0.1 \ - --optimizer nag --clip-norm 0.1 \ - --lr 0.5 --lr-scheduler fixed --force-anneal 50 \ - --max-tokens 3000 \ - --save-dir checkpoints/fconv_wmt_en_fr - -# Evaluate -fairseq-generate \ - data-bin/fconv_wmt_en_fr \ - --path checkpoints/fconv_wmt_en_fr/checkpoint_best.pt \ - --beam 5 --remove-bpe -``` - -## Multilingual Translation - -We also support training multilingual translation models. In this example we'll -train a multilingual `{de,fr}-en` translation model using the IWSLT'17 datasets. - -Note that we use slightly different preprocessing here than for the IWSLT'14 -En-De data above. In particular we learn a joint BPE code for all three -languages and use fairseq-interactive and sacrebleu for scoring the test set. - -```bash -# First install sacrebleu and sentencepiece -pip install sacrebleu sentencepiece - -# Then download and preprocess the data -cd examples/translation/ -bash prepare-iwslt17-multilingual.sh -cd ../.. - -# Binarize the de-en dataset -TEXT=examples/translation/iwslt17.de_fr.en.bpe16k -fairseq-preprocess --source-lang de --target-lang en \ - --trainpref $TEXT/train.bpe.de-en \ - --validpref $TEXT/valid0.bpe.de-en,$TEXT/valid1.bpe.de-en,$TEXT/valid2.bpe.de-en,$TEXT/valid3.bpe.de-en,$TEXT/valid4.bpe.de-en,$TEXT/valid5.bpe.de-en \ - --destdir data-bin/iwslt17.de_fr.en.bpe16k \ - --workers 10 - -# Binarize the fr-en dataset -# NOTE: it's important to reuse the en dictionary from the previous step -fairseq-preprocess --source-lang fr --target-lang en \ - --trainpref $TEXT/train.bpe.fr-en \ - --validpref $TEXT/valid0.bpe.fr-en,$TEXT/valid1.bpe.fr-en,$TEXT/valid2.bpe.fr-en,$TEXT/valid3.bpe.fr-en,$TEXT/valid4.bpe.fr-en,$TEXT/valid5.bpe.fr-en \ - --tgtdict data-bin/iwslt17.de_fr.en.bpe16k/dict.en.txt \ - --destdir data-bin/iwslt17.de_fr.en.bpe16k \ - --workers 10 - -# Train a multilingual transformer model -# NOTE: the command below assumes 1 GPU, but accumulates gradients from -# 8 fwd/bwd passes to simulate training on 8 GPUs -mkdir -p checkpoints/multilingual_transformer -CUDA_VISIBLE_DEVICES=0 fairseq-train data-bin/iwslt17.de_fr.en.bpe16k/ \ - --max-epoch 50 \ - --ddp-backend=legacy_ddp \ - --task multilingual_translation --lang-pairs de-en,fr-en \ - --arch multilingual_transformer_iwslt_de_en \ - --share-decoders --share-decoder-input-output-embed \ - --optimizer adam --adam-betas '(0.9, 0.98)' \ - --lr 0.0005 --lr-scheduler inverse_sqrt \ - --warmup-updates 4000 --warmup-init-lr '1e-07' \ - --label-smoothing 0.1 --criterion label_smoothed_cross_entropy \ - --dropout 0.3 --weight-decay 0.0001 \ - --save-dir checkpoints/multilingual_transformer \ - --max-tokens 4000 \ - --update-freq 8 - -# Generate and score the test set with sacrebleu -SRC=de -sacrebleu --test-set iwslt17 --language-pair ${SRC}-en --echo src \ - | python scripts/spm_encode.py --model examples/translation/iwslt17.de_fr.en.bpe16k/sentencepiece.bpe.model \ - > iwslt17.test.${SRC}-en.${SRC}.bpe -cat iwslt17.test.${SRC}-en.${SRC}.bpe \ - | fairseq-interactive data-bin/iwslt17.de_fr.en.bpe16k/ \ - --task multilingual_translation --lang-pairs de-en,fr-en \ - --source-lang ${SRC} --target-lang en \ - --path checkpoints/multilingual_transformer/checkpoint_best.pt \ - --buffer-size 2000 --batch-size 128 \ - --beam 5 --remove-bpe=sentencepiece \ - > iwslt17.test.${SRC}-en.en.sys -grep ^H iwslt17.test.${SRC}-en.en.sys | cut -f3 \ - | sacrebleu --test-set iwslt17 --language-pair ${SRC}-en -``` - -##### Argument format during inference - -During inference it is required to specify a single `--source-lang` and -`--target-lang`, which indicates the inference langauge direction. -`--lang-pairs`, `--encoder-langtok`, `--decoder-langtok` have to be set to -the same value as training. diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/translation/prepare-iwslt14.sh b/audio/safeear/safeear_repo/fairseq_ours/examples/translation/prepare-iwslt14.sh deleted file mode 100644 index 2fb6643fbccb58701dcbb77d91430e68a821ba38..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/translation/prepare-iwslt14.sh +++ /dev/null @@ -1,115 +0,0 @@ -#!/usr/bin/env bash -# -# Adapted from https://github.com/facebookresearch/MIXER/blob/master/prepareData.sh - -echo 'Cloning Moses github repository (for tokenization scripts)...' -git clone https://github.com/moses-smt/mosesdecoder.git - -echo 'Cloning Subword NMT repository (for BPE pre-processing)...' -git clone https://github.com/rsennrich/subword-nmt.git - -SCRIPTS=mosesdecoder/scripts -TOKENIZER=$SCRIPTS/tokenizer/tokenizer.perl -LC=$SCRIPTS/tokenizer/lowercase.perl -CLEAN=$SCRIPTS/training/clean-corpus-n.perl -BPEROOT=subword-nmt/subword_nmt -BPE_TOKENS=10000 - -URL="http://dl.fbaipublicfiles.com/fairseq/data/iwslt14/de-en.tgz" -GZ=de-en.tgz - -if [ ! -d "$SCRIPTS" ]; then - echo "Please set SCRIPTS variable correctly to point to Moses scripts." - exit -fi - -src=de -tgt=en -lang=de-en -prep=iwslt14.tokenized.de-en -tmp=$prep/tmp -orig=orig - -mkdir -p $orig $tmp $prep - -echo "Downloading data from ${URL}..." -cd $orig -wget "$URL" - -if [ -f $GZ ]; then - echo "Data successfully downloaded." -else - echo "Data not successfully downloaded." - exit -fi - -tar zxvf $GZ -cd .. - -echo "pre-processing train data..." -for l in $src $tgt; do - f=train.tags.$lang.$l - tok=train.tags.$lang.tok.$l - - cat $orig/$lang/$f | \ - grep -v '' | \ - grep -v '' | \ - grep -v '' | \ - sed -e 's///g' | \ - sed -e 's/<\/title>//g' | \ - sed -e 's/<description>//g' | \ - sed -e 's/<\/description>//g' | \ - perl $TOKENIZER -threads 8 -l $l > $tmp/$tok - echo "" -done -perl $CLEAN -ratio 1.5 $tmp/train.tags.$lang.tok $src $tgt $tmp/train.tags.$lang.clean 1 175 -for l in $src $tgt; do - perl $LC < $tmp/train.tags.$lang.clean.$l > $tmp/train.tags.$lang.$l -done - -echo "pre-processing valid/test data..." -for l in $src $tgt; do - for o in `ls $orig/$lang/IWSLT14.TED*.$l.xml`; do - fname=${o##*/} - f=$tmp/${fname%.*} - echo $o $f - grep '<seg id' $o | \ - sed -e 's/<seg id="[0-9]*">\s*//g' | \ - sed -e 's/\s*<\/seg>\s*//g' | \ - sed -e "s/\’/\'/g" | \ - perl $TOKENIZER -threads 8 -l $l | \ - perl $LC > $f - echo "" - done -done - - -echo "creating train, valid, test..." -for l in $src $tgt; do - awk '{if (NR%23 == 0) print $0; }' $tmp/train.tags.de-en.$l > $tmp/valid.$l - awk '{if (NR%23 != 0) print $0; }' $tmp/train.tags.de-en.$l > $tmp/train.$l - - cat $tmp/IWSLT14.TED.dev2010.de-en.$l \ - $tmp/IWSLT14.TEDX.dev2012.de-en.$l \ - $tmp/IWSLT14.TED.tst2010.de-en.$l \ - $tmp/IWSLT14.TED.tst2011.de-en.$l \ - $tmp/IWSLT14.TED.tst2012.de-en.$l \ - > $tmp/test.$l -done - -TRAIN=$tmp/train.en-de -BPE_CODE=$prep/code -rm -f $TRAIN -for l in $src $tgt; do - cat $tmp/train.$l >> $TRAIN -done - -echo "learn_bpe.py on ${TRAIN}..." -python $BPEROOT/learn_bpe.py -s $BPE_TOKENS < $TRAIN > $BPE_CODE - -for L in $src $tgt; do - for f in train.$L valid.$L test.$L; do - echo "apply_bpe.py to ${f}..." - python $BPEROOT/apply_bpe.py -c $BPE_CODE < $tmp/$f > $prep/$f - done -done diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/translation/prepare-iwslt17-multilingual.sh b/audio/safeear/safeear_repo/fairseq_ours/examples/translation/prepare-iwslt17-multilingual.sh deleted file mode 100644 index 23be87555322bc03b13e9d95951d88b1a442f97a..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/translation/prepare-iwslt17-multilingual.sh +++ /dev/null @@ -1,133 +0,0 @@ -#!/bin/bash -# Copyright (c) Facebook, Inc. and its affiliates. -# All rights reserved. -# -# This source code is licensed under the license found in the -# LICENSE file in the root directory of this source tree. - -SRCS=( - "de" - "fr" -) -TGT=en - -ROOT=$(dirname "$0") -SCRIPTS=$ROOT/../../scripts -SPM_TRAIN=$SCRIPTS/spm_train.py -SPM_ENCODE=$SCRIPTS/spm_encode.py - -BPESIZE=16384 -ORIG=$ROOT/iwslt17_orig -DATA=$ROOT/iwslt17.de_fr.en.bpe16k -mkdir -p "$ORIG" "$DATA" - -TRAIN_MINLEN=1 # remove sentences with <1 BPE token -TRAIN_MAXLEN=250 # remove sentences with >250 BPE tokens - -URLS=( - "https://wit3.fbk.eu/archive/2017-01-trnted/texts/de/en/de-en.tgz" - "https://wit3.fbk.eu/archive/2017-01-trnted/texts/fr/en/fr-en.tgz" -) -ARCHIVES=( - "de-en.tgz" - "fr-en.tgz" -) -VALID_SETS=( - "IWSLT17.TED.dev2010.de-en IWSLT17.TED.tst2010.de-en IWSLT17.TED.tst2011.de-en IWSLT17.TED.tst2012.de-en IWSLT17.TED.tst2013.de-en IWSLT17.TED.tst2014.de-en IWSLT17.TED.tst2015.de-en" - "IWSLT17.TED.dev2010.fr-en IWSLT17.TED.tst2010.fr-en IWSLT17.TED.tst2011.fr-en IWSLT17.TED.tst2012.fr-en IWSLT17.TED.tst2013.fr-en IWSLT17.TED.tst2014.fr-en IWSLT17.TED.tst2015.fr-en" -) - -# download and extract data -for ((i=0;i<${#URLS[@]};++i)); do - ARCHIVE=$ORIG/${ARCHIVES[i]} - if [ -f "$ARCHIVE" ]; then - echo "$ARCHIVE already exists, skipping download" - else - URL=${URLS[i]} - wget -P "$ORIG" "$URL" - if [ -f "$ARCHIVE" ]; then - echo "$URL successfully downloaded." - else - echo "$URL not successfully downloaded." - exit 1 - fi - fi - FILE=${ARCHIVE: -4} - if [ -e "$FILE" ]; then - echo "$FILE already exists, skipping extraction" - else - tar -C "$ORIG" -xzvf "$ARCHIVE" - fi -done - -echo "pre-processing train data..." -for SRC in "${SRCS[@]}"; do - for LANG in "${SRC}" "${TGT}"; do - cat "$ORIG/${SRC}-${TGT}/train.tags.${SRC}-${TGT}.${LANG}" \ - | grep -v '<url>' \ - | grep -v '<talkid>' \ - | grep -v '<keywords>' \ - | grep -v '<speaker>' \ - | grep -v '<reviewer' \ - | grep -v '<translator' \ - | grep -v '<doc' \ - | grep -v '</doc>' \ - | sed -e 's/<title>//g' \ - | sed -e 's/<\/title>//g' \ - | sed -e 's/<description>//g' \ - | sed -e 's/<\/description>//g' \ - | sed 's/^\s*//g' \ - | sed 's/\s*$//g' \ - > "$DATA/train.${SRC}-${TGT}.${LANG}" - done -done - -echo "pre-processing valid data..." -for ((i=0;i<${#SRCS[@]};++i)); do - SRC=${SRCS[i]} - VALID_SET=(${VALID_SETS[i]}) - for ((j=0;j<${#VALID_SET[@]};++j)); do - FILE=${VALID_SET[j]} - for LANG in "$SRC" "$TGT"; do - grep '<seg id' "$ORIG/${SRC}-${TGT}/${FILE}.${LANG}.xml" \ - | sed -e 's/<seg id="[0-9]*">\s*//g' \ - | sed -e 's/\s*<\/seg>\s*//g' \ - | sed -e "s/\’/\'/g" \ - > "$DATA/valid${j}.${SRC}-${TGT}.${LANG}" - done - done -done - -# learn BPE with sentencepiece -TRAIN_FILES=$(for SRC in "${SRCS[@]}"; do echo $DATA/train.${SRC}-${TGT}.${SRC}; echo $DATA/train.${SRC}-${TGT}.${TGT}; done | tr "\n" ",") -echo "learning joint BPE over ${TRAIN_FILES}..." -python "$SPM_TRAIN" \ - --input=$TRAIN_FILES \ - --model_prefix=$DATA/sentencepiece.bpe \ - --vocab_size=$BPESIZE \ - --character_coverage=1.0 \ - --model_type=bpe - -# encode train/valid -echo "encoding train with learned BPE..." -for SRC in "${SRCS[@]}"; do - python "$SPM_ENCODE" \ - --model "$DATA/sentencepiece.bpe.model" \ - --output_format=piece \ - --inputs $DATA/train.${SRC}-${TGT}.${SRC} $DATA/train.${SRC}-${TGT}.${TGT} \ - --outputs $DATA/train.bpe.${SRC}-${TGT}.${SRC} $DATA/train.bpe.${SRC}-${TGT}.${TGT} \ - --min-len $TRAIN_MINLEN --max-len $TRAIN_MAXLEN -done - -echo "encoding valid with learned BPE..." -for ((i=0;i<${#SRCS[@]};++i)); do - SRC=${SRCS[i]} - VALID_SET=(${VALID_SETS[i]}) - for ((j=0;j<${#VALID_SET[@]};++j)); do - python "$SPM_ENCODE" \ - --model "$DATA/sentencepiece.bpe.model" \ - --output_format=piece \ - --inputs $DATA/valid${j}.${SRC}-${TGT}.${SRC} $DATA/valid${j}.${SRC}-${TGT}.${TGT} \ - --outputs $DATA/valid${j}.bpe.${SRC}-${TGT}.${SRC} $DATA/valid${j}.bpe.${SRC}-${TGT}.${TGT} - done -done diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/translation/prepare-wmt14en2de.sh b/audio/safeear/safeear_repo/fairseq_ours/examples/translation/prepare-wmt14en2de.sh deleted file mode 100644 index 6702c88b568c9e680b525593ff0c9fb0a474825d..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/translation/prepare-wmt14en2de.sh +++ /dev/null @@ -1,142 +0,0 @@ -#!/bin/bash -# Adapted from https://github.com/facebookresearch/MIXER/blob/master/prepareData.sh - -echo 'Cloning Moses github repository (for tokenization scripts)...' -git clone https://github.com/moses-smt/mosesdecoder.git - -echo 'Cloning Subword NMT repository (for BPE pre-processing)...' -git clone https://github.com/rsennrich/subword-nmt.git - -SCRIPTS=mosesdecoder/scripts -TOKENIZER=$SCRIPTS/tokenizer/tokenizer.perl -CLEAN=$SCRIPTS/training/clean-corpus-n.perl -NORM_PUNC=$SCRIPTS/tokenizer/normalize-punctuation.perl -REM_NON_PRINT_CHAR=$SCRIPTS/tokenizer/remove-non-printing-char.perl -BPEROOT=subword-nmt/subword_nmt -BPE_TOKENS=40000 - -URLS=( - "http://statmt.org/wmt13/training-parallel-europarl-v7.tgz" - "http://statmt.org/wmt13/training-parallel-commoncrawl.tgz" - "http://data.statmt.org/wmt17/translation-task/training-parallel-nc-v12.tgz" - "http://data.statmt.org/wmt17/translation-task/dev.tgz" - "http://statmt.org/wmt14/test-full.tgz" -) -FILES=( - "training-parallel-europarl-v7.tgz" - "training-parallel-commoncrawl.tgz" - "training-parallel-nc-v12.tgz" - "dev.tgz" - "test-full.tgz" -) -CORPORA=( - "training/europarl-v7.de-en" - "commoncrawl.de-en" - "training/news-commentary-v12.de-en" -) - -# This will make the dataset compatible to the one used in "Convolutional Sequence to Sequence Learning" -# https://arxiv.org/abs/1705.03122 -if [ "$1" == "--icml17" ]; then - URLS[2]="http://statmt.org/wmt14/training-parallel-nc-v9.tgz" - FILES[2]="training-parallel-nc-v9.tgz" - CORPORA[2]="training/news-commentary-v9.de-en" - OUTDIR=wmt14_en_de -else - OUTDIR=wmt17_en_de -fi - -if [ ! -d "$SCRIPTS" ]; then - echo "Please set SCRIPTS variable correctly to point to Moses scripts." - exit -fi - -src=en -tgt=de -lang=en-de -prep=$OUTDIR -tmp=$prep/tmp -orig=orig -dev=dev/newstest2013 - -mkdir -p $orig $tmp $prep - -cd $orig - -for ((i=0;i<${#URLS[@]};++i)); do - file=${FILES[i]} - if [ -f $file ]; then - echo "$file already exists, skipping download" - else - url=${URLS[i]} - wget "$url" - if [ -f $file ]; then - echo "$url successfully downloaded." - else - echo "$url not successfully downloaded." - exit -1 - fi - if [ ${file: -4} == ".tgz" ]; then - tar zxvf $file - elif [ ${file: -4} == ".tar" ]; then - tar xvf $file - fi - fi -done -cd .. - -echo "pre-processing train data..." -for l in $src $tgt; do - rm $tmp/train.tags.$lang.tok.$l - for f in "${CORPORA[@]}"; do - cat $orig/$f.$l | \ - perl $NORM_PUNC $l | \ - perl $REM_NON_PRINT_CHAR | \ - perl $TOKENIZER -threads 8 -a -l $l >> $tmp/train.tags.$lang.tok.$l - done -done - -echo "pre-processing test data..." -for l in $src $tgt; do - if [ "$l" == "$src" ]; then - t="src" - else - t="ref" - fi - grep '<seg id' $orig/test-full/newstest2014-deen-$t.$l.sgm | \ - sed -e 's/<seg id="[0-9]*">\s*//g' | \ - sed -e 's/\s*<\/seg>\s*//g' | \ - sed -e "s/\’/\'/g" | \ - perl $TOKENIZER -threads 8 -a -l $l > $tmp/test.$l - echo "" -done - -echo "splitting train and valid..." -for l in $src $tgt; do - awk '{if (NR%100 == 0) print $0; }' $tmp/train.tags.$lang.tok.$l > $tmp/valid.$l - awk '{if (NR%100 != 0) print $0; }' $tmp/train.tags.$lang.tok.$l > $tmp/train.$l -done - -TRAIN=$tmp/train.de-en -BPE_CODE=$prep/code -rm -f $TRAIN -for l in $src $tgt; do - cat $tmp/train.$l >> $TRAIN -done - -echo "learn_bpe.py on ${TRAIN}..." -python $BPEROOT/learn_bpe.py -s $BPE_TOKENS < $TRAIN > $BPE_CODE - -for L in $src $tgt; do - for f in train.$L valid.$L test.$L; do - echo "apply_bpe.py to ${f}..." - python $BPEROOT/apply_bpe.py -c $BPE_CODE < $tmp/$f > $tmp/bpe.$f - done -done - -perl $CLEAN -ratio 1.5 $tmp/bpe.train $src $tgt $prep/train 1 250 -perl $CLEAN -ratio 1.5 $tmp/bpe.valid $src $tgt $prep/valid 1 250 - -for L in $src $tgt; do - cp $tmp/bpe.test.$L $prep/test.$L -done diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/translation/prepare-wmt14en2fr.sh b/audio/safeear/safeear_repo/fairseq_ours/examples/translation/prepare-wmt14en2fr.sh deleted file mode 100644 index 2ac97a5b76fab255449493488ed8bd67350a7bac..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/translation/prepare-wmt14en2fr.sh +++ /dev/null @@ -1,136 +0,0 @@ -#!/bin/bash -# Adapted from https://github.com/facebookresearch/MIXER/blob/master/prepareData.sh - -echo 'Cloning Moses github repository (for tokenization scripts)...' -git clone https://github.com/moses-smt/mosesdecoder.git - -echo 'Cloning Subword NMT repository (for BPE pre-processing)...' -git clone https://github.com/rsennrich/subword-nmt.git - -SCRIPTS=mosesdecoder/scripts -TOKENIZER=$SCRIPTS/tokenizer/tokenizer.perl -CLEAN=$SCRIPTS/training/clean-corpus-n.perl -NORM_PUNC=$SCRIPTS/tokenizer/normalize-punctuation.perl -REM_NON_PRINT_CHAR=$SCRIPTS/tokenizer/remove-non-printing-char.perl -BPEROOT=subword-nmt/subword_nmt -BPE_TOKENS=40000 - -URLS=( - "http://statmt.org/wmt13/training-parallel-europarl-v7.tgz" - "http://statmt.org/wmt13/training-parallel-commoncrawl.tgz" - "http://statmt.org/wmt13/training-parallel-un.tgz" - "http://statmt.org/wmt14/training-parallel-nc-v9.tgz" - "http://statmt.org/wmt10/training-giga-fren.tar" - "http://statmt.org/wmt14/test-full.tgz" -) -FILES=( - "training-parallel-europarl-v7.tgz" - "training-parallel-commoncrawl.tgz" - "training-parallel-un.tgz" - "training-parallel-nc-v9.tgz" - "training-giga-fren.tar" - "test-full.tgz" -) -CORPORA=( - "training/europarl-v7.fr-en" - "commoncrawl.fr-en" - "un/undoc.2000.fr-en" - "training/news-commentary-v9.fr-en" - "giga-fren.release2.fixed" -) - -if [ ! -d "$SCRIPTS" ]; then - echo "Please set SCRIPTS variable correctly to point to Moses scripts." - exit -fi - -src=en -tgt=fr -lang=en-fr -prep=wmt14_en_fr -tmp=$prep/tmp -orig=orig - -mkdir -p $orig $tmp $prep - -cd $orig - -for ((i=0;i<${#URLS[@]};++i)); do - file=${FILES[i]} - if [ -f $file ]; then - echo "$file already exists, skipping download" - else - url=${URLS[i]} - wget "$url" - if [ -f $file ]; then - echo "$url successfully downloaded." - else - echo "$url not successfully downloaded." - exit -1 - fi - if [ ${file: -4} == ".tgz" ]; then - tar zxvf $file - elif [ ${file: -4} == ".tar" ]; then - tar xvf $file - fi - fi -done - -gunzip giga-fren.release2.fixed.*.gz -cd .. - -echo "pre-processing train data..." -for l in $src $tgt; do - rm $tmp/train.tags.$lang.tok.$l - for f in "${CORPORA[@]}"; do - cat $orig/$f.$l | \ - perl $NORM_PUNC $l | \ - perl $REM_NON_PRINT_CHAR | \ - perl $TOKENIZER -threads 8 -a -l $l >> $tmp/train.tags.$lang.tok.$l - done -done - -echo "pre-processing test data..." -for l in $src $tgt; do - if [ "$l" == "$src" ]; then - t="src" - else - t="ref" - fi - grep '<seg id' $orig/test-full/newstest2014-fren-$t.$l.sgm | \ - sed -e 's/<seg id="[0-9]*">\s*//g' | \ - sed -e 's/\s*<\/seg>\s*//g' | \ - sed -e "s/\’/\'/g" | \ - perl $TOKENIZER -threads 8 -a -l $l > $tmp/test.$l - echo "" -done - -echo "splitting train and valid..." -for l in $src $tgt; do - awk '{if (NR%1333 == 0) print $0; }' $tmp/train.tags.$lang.tok.$l > $tmp/valid.$l - awk '{if (NR%1333 != 0) print $0; }' $tmp/train.tags.$lang.tok.$l > $tmp/train.$l -done - -TRAIN=$tmp/train.fr-en -BPE_CODE=$prep/code -rm -f $TRAIN -for l in $src $tgt; do - cat $tmp/train.$l >> $TRAIN -done - -echo "learn_bpe.py on ${TRAIN}..." -python $BPEROOT/learn_bpe.py -s $BPE_TOKENS < $TRAIN > $BPE_CODE - -for L in $src $tgt; do - for f in train.$L valid.$L test.$L; do - echo "apply_bpe.py to ${f}..." - python $BPEROOT/apply_bpe.py -c $BPE_CODE < $tmp/$f > $tmp/bpe.$f - done -done - -perl $CLEAN -ratio 1.5 $tmp/bpe.train $src $tgt $prep/train 1 250 -perl $CLEAN -ratio 1.5 $tmp/bpe.valid $src $tgt $prep/valid 1 250 - -for L in $src $tgt; do - cp $tmp/bpe.test.$L $prep/test.$L -done diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/translation_moe/README.md b/audio/safeear/safeear_repo/fairseq_ours/examples/translation_moe/README.md deleted file mode 100644 index 2e5c8af617f410f64ca38d29447bd05b6af8c5a8..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/translation_moe/README.md +++ /dev/null @@ -1,89 +0,0 @@ -# Mixture Models for Diverse Machine Translation: Tricks of the Trade (Shen et al., 2019) - -This page includes instructions for reproducing results from the paper [Mixture Models for Diverse Machine Translation: Tricks of the Trade (Shen et al., 2019)](https://arxiv.org/abs/1902.07816). - -## Download data - -First, follow the [instructions to download and preprocess the WMT'17 En-De dataset](../translation#prepare-wmt14en2desh). -Make sure to learn a joint vocabulary by passing the `--joined-dictionary` option to `fairseq-preprocess`. - -## Train a model - -Then we can train a mixture of experts model using the `translation_moe` task. -Use the `--method` flag to choose the MoE variant; we support hard mixtures with a learned or uniform prior (`--method hMoElp` and `hMoEup`, respectively) and soft mixures (`--method sMoElp` and `sMoEup`). -The model is trained with online responsibility assignment and shared parameterization. - -The following command will train a `hMoElp` model with `3` experts: -```bash -fairseq-train --ddp-backend='legacy_ddp' \ - data-bin/wmt17_en_de \ - --max-update 100000 \ - --task translation_moe --user-dir examples/translation_moe/translation_moe_src \ - --method hMoElp --mean-pool-gating-network \ - --num-experts 3 \ - --arch transformer_wmt_en_de --share-all-embeddings \ - --optimizer adam --adam-betas '(0.9, 0.98)' --clip-norm 0.0 \ - --lr-scheduler inverse_sqrt --warmup-init-lr 1e-07 --warmup-updates 4000 \ - --lr 0.0007 \ - --dropout 0.1 --weight-decay 0.0 --criterion cross_entropy \ - --max-tokens 3584 -``` - -## Translate - -Once a model is trained, we can generate translations from different experts using the `--gen-expert` option. -For example, to generate from expert 0: -```bash -fairseq-generate data-bin/wmt17_en_de \ - --path checkpoints/checkpoint_best.pt \ - --beam 1 --remove-bpe \ - --task translation_moe --user-dir examples/translation_moe/translation_moe_src \ - --method hMoElp --mean-pool-gating-network \ - --num-experts 3 \ - --gen-expert 0 -``` - -## Evaluate - -First download a tokenized version of the WMT'14 En-De test set with multiple references: -```bash -wget dl.fbaipublicfiles.com/fairseq/data/wmt14-en-de.extra_refs.tok -``` - -Next apply BPE on the fly and run generation for each expert: -```bash -BPE_CODE=examples/translation/wmt17_en_de/code -for EXPERT in $(seq 0 2); do \ - cat wmt14-en-de.extra_refs.tok \ - | grep ^S | cut -f 2 \ - | fairseq-interactive data-bin/wmt17_en_de \ - --path checkpoints/checkpoint_best.pt \ - --beam 1 \ - --bpe subword_nmt --bpe-codes $BPE_CODE \ - --buffer-size 500 --max-tokens 6000 \ - --task translation_moe --user-dir examples/translation_moe/translation_moe_src \ - --method hMoElp --mean-pool-gating-network \ - --num-experts 3 \ - --gen-expert $EXPERT ; \ -done > wmt14-en-de.extra_refs.tok.gen.3experts -``` - -Finally use `score_moe.py` to compute pairwise BLUE and average oracle BLEU: -```bash -python examples/translation_moe/score.py --sys wmt14-en-de.extra_refs.tok.gen.3experts --ref wmt14-en-de.extra_refs.tok -# pairwise BLEU: 48.26 -# #refs covered: 2.11 -# multi-reference BLEU (leave-one-out): 59.46 -``` -This matches row 3 from Table 7 in the paper. - -## Citation - -```bibtex -@article{shen2019mixture, - title = {Mixture Models for Diverse Machine Translation: Tricks of the Trade}, - author = {Tianxiao Shen and Myle Ott and Michael Auli and Marc'Aurelio Ranzato}, - journal = {International Conference on Machine Learning}, - year = 2019, -} -``` diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/translation_moe/score.py b/audio/safeear/safeear_repo/fairseq_ours/examples/translation_moe/score.py deleted file mode 100644 index fedb0a55ebc028bec1f170bcdafb3f0d9a2ed2b0..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/translation_moe/score.py +++ /dev/null @@ -1,198 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. -""" -Scoring script for computing pairwise BLEU and multi-ref BLEU over a set of -candidate hypotheses. - -See `"Mixture Models for Diverse Machine Translation: Tricks of the Trade" -(Shen et al., 2019) <https://arxiv.org/abs/1902.07816>`_. -""" - -import argparse -import random -import sys -from itertools import chain - -import numpy as np -import sacrebleu -from sacrebleu import corpus_bleu as _corpus_bleu - - -def main(): - parser = argparse.ArgumentParser(sys.argv[0]) - parser.add_argument( - "--sys", nargs="*", default="", metavar="FILE", help="path to system output" - ) - parser.add_argument("--ref", default="", metavar="FILE", help="path to references") - parser.add_argument( - "--output", - default="", - metavar="FILE", - help="print outputs into a pretty format", - ) - args = parser.parse_args() - - if args.sys: - src, tgt, hypos, log_probs = load_sys(args.sys) - print("pairwise BLEU: %.2f" % pairwise(hypos)) - if args.output: - merge(src, tgt, hypos, log_probs, args.output) - - if args.ref: - _, _, refs = load_ref(args.ref) - if args.sys: - multi_ref(refs, hypos) - else: - intra_ref(refs) - - -def dictolist(d): - a = sorted(d.items(), key=lambda i: i[0]) - return [i[1] for i in a] - - -def load_sys(paths): - src, tgt, hypos, log_probs = {}, {}, {}, {} - for path in paths: - with open(path) as f: - for line in f: - line = line.rstrip() - # S: source - # T: target - # D: detokenized system output - if line.startswith(("S-", "T-", "D-")): - i = int(line[line.find("-") + 1 : line.find("\t")]) - if line.startswith("S-"): - src[i] = line.split("\t")[1] - if line.startswith("T-"): - tgt[i] = line.split("\t")[1] - if line.startswith("D-"): - if i not in hypos: - hypos[i] = [] - log_probs[i] = [] - hypos[i].append(line.split("\t")[2]) - log_probs[i].append(float(line.split("\t")[1])) - return dictolist(src), dictolist(tgt), dictolist(hypos), dictolist(log_probs) - - -def load_ref(path): - with open(path) as f: - lines = f.readlines() - src, tgt, refs = [], [], [] - i = 0 - while i < len(lines): - if lines[i].startswith("S-"): - src.append(lines[i].split("\t")[1].rstrip()) - i += 1 - elif lines[i].startswith("T-"): - tgt.append(lines[i].split("\t")[1].rstrip()) - i += 1 - else: - a = [] - while i < len(lines) and lines[i].startswith("R"): - a.append(lines[i].split("\t")[1].rstrip()) - i += 1 - refs.append(a) - return src, tgt, refs - - -def merge(src, tgt, hypos, log_probs, path): - with open(path, "w") as f: - for s, t, hs, lps in zip(src, tgt, hypos, log_probs): - f.write(s + "\n") - f.write(t + "\n") - f.write("\n") - for h, lp in zip(hs, lps): - f.write("\t%f\t%s\n" % (lp, h.strip())) - f.write("------------------------------------------------------\n") - - -def corpus_bleu(sys_stream, ref_streams): - bleu = _corpus_bleu(sys_stream, ref_streams, tokenize="none") - return bleu.score - - -def sentence_bleu(hypothesis, reference): - bleu = _corpus_bleu(hypothesis, reference) - for i in range(1, 4): - bleu.counts[i] += 1 - bleu.totals[i] += 1 - bleu = sacrebleu.BLEU.compute_bleu( - bleu.counts, - bleu.totals, - bleu.sys_len, - bleu.ref_len, - smooth_method="exp", - ) - return bleu.score - - -def pairwise(sents): - _ref, _hypo = [], [] - for s in sents: - for i in range(len(s)): - for j in range(len(s)): - if i != j: - _ref.append(s[i]) - _hypo.append(s[j]) - return corpus_bleu(_hypo, [_ref]) - - -def multi_ref(refs, hypos): - _ref, _hypo = [], [] - ref_cnt = 0 - assert len(refs) == len(hypos) - - # count number of refs covered - for rs, hs in zip(refs, hypos): - a = set() - for h in hs: - s = [sentence_bleu(h, r) for r in rs] - j = np.argmax(s) - _ref.append(rs[j]) - _hypo.append(h) - best = [k for k in range(len(rs)) if s[k] == s[j]] - a.add(random.choice(best)) - ref_cnt += len(a) - print("#refs covered: %.2f" % (ref_cnt / len(refs))) - - # transpose refs and hypos - refs = list(zip(*refs)) - hypos = list(zip(*hypos)) - - # compute multi-ref corpus BLEU (leave-one-out to be comparable to intra_ref) - k = len(hypos) - m = len(refs) - flat_hypos = [hypos[j][i] for i in range(len(hypos[0])) for j in range(k)] - duplicated_refs = [[ref for ref in refs_i for _ in range(k)] for refs_i in refs] - loo_bleus = [] - for held_out_ref in range(m): - remaining_refs = ( - duplicated_refs[:held_out_ref] + duplicated_refs[held_out_ref + 1 :] - ) - assert len(remaining_refs) == m - 1 - loo_bleus.append(corpus_bleu(flat_hypos, remaining_refs)) - print("average multi-reference BLEU (leave-one-out): %.2f" % np.mean(loo_bleus)) - - -def intra_ref(refs): - print("ref pairwise BLEU: %.2f" % pairwise(refs)) - refs = list(zip(*refs)) - m = len(refs) - concat_h = [] - concat_rest = [[] for j in range(m - 1)] - for i, h in enumerate(refs): - rest = refs[:i] + refs[i + 1 :] - concat_h.append(h) - for j in range(m - 1): - concat_rest[j].extend(rest[j]) - concat_h = list(chain.from_iterable(concat_h)) - bleu = corpus_bleu(concat_h, concat_rest) - print("multi-reference BLEU (leave-one-out): %.2f" % bleu) - - -if __name__ == "__main__": - main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/translation_moe/translation_moe_src/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/examples/translation_moe/translation_moe_src/__init__.py deleted file mode 100644 index c0abe53e973b4bb31cfb062708965d002c79b6e7..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/translation_moe/translation_moe_src/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from . import translation_moe # noqa diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/translation_moe/translation_moe_src/logsumexp_moe.py b/audio/safeear/safeear_repo/fairseq_ours/examples/translation_moe/translation_moe_src/logsumexp_moe.py deleted file mode 100644 index fb299daecbc2b15fb66555bbfb8d1d983e481518..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/translation_moe/translation_moe_src/logsumexp_moe.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import torch - - -class LogSumExpMoE(torch.autograd.Function): - """Standard LogSumExp forward pass, but use *posterior* for the backward. - - See `"Mixture Models for Diverse Machine Translation: Tricks of the Trade" - (Shen et al., 2019) <https://arxiv.org/abs/1902.07816>`_. - """ - - @staticmethod - def forward(ctx, logp, posterior, dim=-1): - ctx.save_for_backward(posterior) - ctx.dim = dim - return torch.logsumexp(logp, dim=dim) - - @staticmethod - def backward(ctx, grad_output): - (posterior,) = ctx.saved_tensors - grad_logp = grad_output.unsqueeze(ctx.dim) * posterior - return grad_logp, None, None diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/translation_moe/translation_moe_src/mean_pool_gating_network.py b/audio/safeear/safeear_repo/fairseq_ours/examples/translation_moe/translation_moe_src/mean_pool_gating_network.py deleted file mode 100644 index efc7ae40bf8fed6c2384cbc6f94477c4caa4c10c..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/translation_moe/translation_moe_src/mean_pool_gating_network.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import torch -import torch.nn.functional as F - - -class MeanPoolGatingNetwork(torch.nn.Module): - """A simple mean-pooling gating network for selecting experts. - - This module applies mean pooling over an encoder's output and returns - reponsibilities for each expert. The encoder format is expected to match - :class:`fairseq.models.transformer.TransformerEncoder`. - """ - - def __init__(self, embed_dim, num_experts, dropout=None): - super().__init__() - self.embed_dim = embed_dim - self.num_experts = num_experts - - self.fc1 = torch.nn.Linear(embed_dim, embed_dim) - self.dropout = torch.nn.Dropout(dropout) if dropout is not None else None - self.fc2 = torch.nn.Linear(embed_dim, num_experts) - - def forward(self, encoder_out): - if not ( - "encoder_out" in encoder_out - and "encoder_padding_mask" in encoder_out - and encoder_out["encoder_out"][0].size(2) == self.embed_dim - ): - raise ValueError("Unexpected format for encoder_out") - - # mean pooling over time - encoder_padding_mask = encoder_out["encoder_padding_mask"][0] # B x T - encoder_out = encoder_out["encoder_out"][0].transpose(0, 1) # B x T x C - if encoder_padding_mask is not None: - encoder_out = encoder_out.clone() # required because of transpose above - encoder_out[encoder_padding_mask] = 0 - ntokens = torch.sum(~encoder_padding_mask, dim=1, keepdim=True) - x = torch.sum(encoder_out, dim=1) / ntokens.type_as(encoder_out) - else: - x = torch.mean(encoder_out, dim=1) - - x = torch.tanh(self.fc1(x)) - if self.dropout is not None: - x = self.dropout(x) - x = self.fc2(x) - return F.log_softmax(x, dim=-1, dtype=torch.float32).type_as(x) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/translation_moe/translation_moe_src/translation_moe.py b/audio/safeear/safeear_repo/fairseq_ours/examples/translation_moe/translation_moe_src/translation_moe.py deleted file mode 100644 index 8ac9e5a3d57408ae98140830293ed60cade840db..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/translation_moe/translation_moe_src/translation_moe.py +++ /dev/null @@ -1,257 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from dataclasses import dataclass, field - -import torch -from fairseq import metrics, utils -from fairseq.dataclass import ChoiceEnum -from fairseq.tasks import register_task -from fairseq.tasks.translation import TranslationConfig, TranslationTask -from omegaconf import II - -from .logsumexp_moe import LogSumExpMoE -from .mean_pool_gating_network import MeanPoolGatingNetwork - -METHOD_CHOICES = ChoiceEnum(["sMoElp", "sMoEup", "hMoElp", "hMoEup"]) - - -@dataclass -class TranslationMoEConfig(TranslationConfig): - method: METHOD_CHOICES = field( - default="hMoEup", - metadata={"help": "MoE method"}, - ) - num_experts: int = field( - default=3, - metadata={"help": "number of experts"}, - ) - mean_pool_gating_network: bool = field( - default=False, - metadata={"help": "use a simple mean-pooling gating network"}, - ) - mean_pool_gating_network_dropout: float = field( - default=0, - metadata={"help": "dropout for mean-pooling gating network"}, - ) - mean_pool_gating_network_encoder_dim: int = field( - default=0, - metadata={"help": "encoder output dim for mean-pooling gating network"}, - ) - gen_expert: int = field( - default=0, - metadata={"help": "which expert to use for generation"}, - ) - sentence_avg: bool = II("optimization.sentence_avg") - - -@register_task("translation_moe", dataclass=TranslationMoEConfig) -class TranslationMoETask(TranslationTask): - """ - Translation task for Mixture of Experts (MoE) models. - - See `"Mixture Models for Diverse Machine Translation: Tricks of the Trade" - (Shen et al., 2019) <https://arxiv.org/abs/1902.07816>`_. - - Args: - src_dict (~fairseq.data.Dictionary): dictionary for the source language - tgt_dict (~fairseq.data.Dictionary): dictionary for the target language - - .. note:: - - The translation task is compatible with :mod:`fairseq-train`, - :mod:`fairseq-generate` and :mod:`fairseq-interactive`. - - The translation task provides the following additional command-line - arguments: - - .. argparse:: - :ref: fairseq.tasks.translation_parser - :prog: - """ - - cfg: TranslationMoEConfig - - def __init__(self, cfg: TranslationMoEConfig, src_dict, tgt_dict): - if cfg.method == "sMoElp": - # soft MoE with learned prior - self.uniform_prior = False - self.hard_selection = False - elif cfg.method == "sMoEup": - # soft MoE with uniform prior - self.uniform_prior = True - self.hard_selection = False - elif cfg.method == "hMoElp": - # hard MoE with learned prior - self.uniform_prior = False - self.hard_selection = True - elif cfg.method == "hMoEup": - # hard MoE with uniform prior - self.uniform_prior = True - self.hard_selection = True - - # add indicator tokens for each expert - for i in range(cfg.num_experts): - # add to both dictionaries in case we're sharing embeddings - src_dict.add_symbol("<expert_{}>".format(i)) - tgt_dict.add_symbol("<expert_{}>".format(i)) - - super().__init__(cfg, src_dict, tgt_dict) - - def build_model(self, cfg, from_checkpoint=False): - from fairseq import models - - model = models.build_model(cfg, self) - if not self.uniform_prior and not hasattr(model, "gating_network"): - if self.cfg.mean_pool_gating_network: - if self.cfg.mean_pool_gating_network_encoder_dim > 0: - encoder_dim = self.cfg.mean_pool_gating_network_encoder_dim - elif getattr(cfg, "encoder_embed_dim", None): - # assume that encoder_embed_dim is the encoder's output dimension - encoder_dim = cfg.encoder_embed_dim - else: - raise ValueError( - "Must specify --mean-pool-gating-network-encoder-dim" - ) - - if self.cfg.mean_pool_gating_network_dropout > 0: - dropout = self.cfg.mean_pool_gating_network_dropout - elif getattr(cfg, "dropout", None): - dropout = cfg.dropout - else: - raise ValueError("Must specify task.mean_pool_gating_network_dropout") - - model.gating_network = MeanPoolGatingNetwork( - encoder_dim, - self.cfg.num_experts, - dropout, - ) - else: - raise ValueError( - "translation_moe task with learned prior requires the model to " - "have a gating network; try using --mean-pool-gating-network" - ) - return model - - def expert_index(self, i): - return i + self.tgt_dict.index("<expert_0>") - - def _get_loss(self, sample, model, criterion): - assert hasattr( - criterion, "compute_loss" - ), "translation_moe task requires the criterion to implement the compute_loss() method" - - k = self.cfg.num_experts - bsz = sample["target"].size(0) - - def get_lprob_y(encoder_out, prev_output_tokens_k): - net_output = model.decoder( - prev_output_tokens=prev_output_tokens_k, - encoder_out=encoder_out, - ) - loss, _ = criterion.compute_loss(model, net_output, sample, reduce=False) - loss = loss.view(bsz, -1) - return -loss.sum(dim=1, keepdim=True) # -> B x 1 - - def get_lprob_yz(winners=None): - encoder_out = model.encoder( - src_tokens=sample["net_input"]["src_tokens"], - src_lengths=sample["net_input"]["src_lengths"], - ) - - if winners is None: - lprob_y = [] - for i in range(k): - prev_output_tokens_k = sample["net_input"][ - "prev_output_tokens" - ].clone() - assert not prev_output_tokens_k.requires_grad - prev_output_tokens_k[:, 0] = self.expert_index(i) - lprob_y.append(get_lprob_y(encoder_out, prev_output_tokens_k)) - lprob_y = torch.cat(lprob_y, dim=1) # -> B x K - else: - prev_output_tokens_k = sample["net_input"]["prev_output_tokens"].clone() - prev_output_tokens_k[:, 0] = self.expert_index(winners) - lprob_y = get_lprob_y(encoder_out, prev_output_tokens_k) # -> B - - if self.uniform_prior: - lprob_yz = lprob_y - else: - lprob_z = model.gating_network(encoder_out) # B x K - if winners is not None: - lprob_z = lprob_z.gather(dim=1, index=winners.unsqueeze(-1)) - lprob_yz = lprob_y + lprob_z.type_as(lprob_y) # B x K - - return lprob_yz - - # compute responsibilities without dropout - with utils.model_eval(model): # disable dropout - with torch.no_grad(): # disable autograd - lprob_yz = get_lprob_yz() # B x K - prob_z_xy = torch.nn.functional.softmax(lprob_yz, dim=1) - assert not prob_z_xy.requires_grad - - # compute loss with dropout - if self.hard_selection: - winners = prob_z_xy.max(dim=1)[1] - loss = -get_lprob_yz(winners) - else: - lprob_yz = get_lprob_yz() # B x K - loss = -LogSumExpMoE.apply(lprob_yz, prob_z_xy, 1) - - loss = loss.sum() - sample_size = ( - sample["target"].size(0) if self.cfg.sentence_avg else sample["ntokens"] - ) - logging_output = { - "loss": utils.item(loss.data), - "ntokens": sample["ntokens"], - "nsentences": bsz, - "sample_size": sample_size, - "posterior": prob_z_xy.float().sum(dim=0).cpu(), - } - return loss, sample_size, logging_output - - def train_step( - self, sample, model, criterion, optimizer, update_num, ignore_grad=False - ): - model.train() - loss, sample_size, logging_output = self._get_loss(sample, model, criterion) - if ignore_grad: - loss *= 0 - optimizer.backward(loss) - return loss, sample_size, logging_output - - def valid_step(self, sample, model, criterion): - model.eval() - with torch.no_grad(): - loss, sample_size, logging_output = self._get_loss(sample, model, criterion) - return loss, sample_size, logging_output - - def inference_step( - self, - generator, - models, - sample, - prefix_tokens=None, - expert=None, - constraints=None, - ): - expert = expert or self.cfg.gen_expert - with torch.no_grad(): - return generator.generate( - models, - sample, - prefix_tokens=prefix_tokens, - constraints=constraints, - bos_token=self.expert_index(expert), - ) - - def reduce_metrics(self, logging_outputs, criterion): - super().reduce_metrics(logging_outputs, criterion) - metrics.log_scalar( - "posterior", - sum(log["posterior"] for log in logging_outputs if "posterior" in log), - ) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/truncated_bptt/README.md b/audio/safeear/safeear_repo/fairseq_ours/examples/truncated_bptt/README.md deleted file mode 100644 index 86518c9d5ef09fbd4fed1512a52e9431b74f08fa..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/truncated_bptt/README.md +++ /dev/null @@ -1,70 +0,0 @@ -# Truncated Backpropagation Through Time (BPTT) - -Truncated BPTT is a useful technique for training language models on very long -sequences. Typically a long sequences is split into chunks and a language model -is trained over the chunks sequentially. The LM may condition on previous -chunks, but gradients only flow through the current chunk. This technique was -the basis for the paper: [Transformer-XL: Attentive Language Models Beyond a -Fixed-Length Context](https://arxiv.org/abs/1901.02860), which achieved -state-of-the-art language modeling results at the time of publication. - -It is slightly tricky to implement Truncated BPTT efficiently in fairseq, since -we need to iterate over the data sequentially and disable any batch shuffling -logic. The code provided in this example illustrates how to implement Truncated -BPTT in fairseq by overriding ``FairseqTask::get_batch_iterator`` to iterate -over the data sequentially. Crucially, this example supports batching and -multi-GPU (data parallel) training. - -##### 0. Setup - -First, see the general [language modeling README](README.md) for instructions on -preprocessing the WikiText-103 data. - -##### 1. Train a Transformer-XL model on WikiText-103 - -We will train a 16-layer Transformer-XL model following the [hyperparameters -used in the original -paper](https://github.com/kimiyoung/transformer-xl/blob/master/pytorch/run_wt103_base.sh). - -The following command assumes 4 GPUs, so that the total batch size is 60 -sequences (15 x 4). Training should take ~24 hours on 4 V100 GPUs: -```bash -CUDA_VISIBLE_DEVICES=0,1,2,3 fairseq-train \ - --user-dir examples/truncated_bptt \ - data-bin/wikitext-103/ \ - --task truncated_bptt_lm --tokens-per-sample 150 \ - --batch-size 15 --max-update 200000 \ - --arch transformer_xl --n-layer 16 --d-model 410 --n-head 10 \ - --d-head 41 --d-inner 2100 --dropout 0.1 --dropatt 0.0 --mem-len 150 \ - --optimizer adam --clip-norm 0.25 \ - --lr-scheduler cosine --warmup-updates 0 --min-lr 0.0 --lr 0.00025 \ - --log-format json --log-interval 25 \ - --fp16 -``` - -If training on a single GPU, set `--update-freq=4` to accumulate 4x gradients -and simulate training on 4 GPUs. - -##### 2. Evaluate - -```bash -fairseq-eval-lm data-bin/wikitext-103/ \ - --path checkpoints/checkpoint_best.pt \ - --user-dir examples/truncated_bptt/ \ - --task truncated_bptt_lm \ - --batch-size 1 --required-batch-size-multiple 1 \ - --model-overrides '{"mem_len":640,"clamp_len":400,"same_length":True}' \ - --tokens-per-sample 64 -# ... | INFO | fairseq_cli.eval_lm | num. model params: 151123537 -# ... | INFO | fairseq_cli.eval_lm | Evaluated 245569 tokens in 83.1s (2956.82 tokens/s) -# ... | INFO | fairseq_cli.eval_lm | Loss (base 2): 4.5668, Perplexity: 23.70 -# Compare to 24.0 test perplexity from the paper -``` - -*Note:* During training the model saw 150 tokens of context -(``--tokens-per-sample=150``) and 150 extra memory tokens (``--mem-len=150``). -During evaluation we measure perplexity on sequences of 64 tokens -(``--tokens-per-sample=64``) and increase the memory length -(``--model-overrides='{"mem_len":640}'``). These settings match the evaluation -settings from [the original -paper](https://github.com/kimiyoung/transformer-xl/blob/master/pytorch/run_wt103_base.sh). diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/truncated_bptt/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/examples/truncated_bptt/__init__.py deleted file mode 100644 index eee484d427a68828462469d133144a8d7c052c40..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/truncated_bptt/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from . import transformer_xl_model, truncated_bptt_lm_task # noqa diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/truncated_bptt/transformer_xl_model.py b/audio/safeear/safeear_repo/fairseq_ours/examples/truncated_bptt/transformer_xl_model.py deleted file mode 100644 index 1e61b5a05930155ba167c7ba270dc46dfdfab5c6..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/truncated_bptt/transformer_xl_model.py +++ /dev/null @@ -1,142 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import logging -from dataclasses import dataclass, field -from typing import Dict, List, Optional - -import torch -from fairseq.dataclass import FairseqDataclass -from fairseq.models import ( - FairseqIncrementalDecoder, - FairseqLanguageModel, - register_model, -) -from fairseq.modules.checkpoint_activations import checkpoint_wrapper -from omegaconf import II - -logger = logging.getLogger(__name__) - - -@dataclass -class TransformerXLConfig(FairseqDataclass): - # defaults come from the original Transformer-XL code - cutoffs: List[int] = field(default_factory=lambda: [20000, 40000, 200000]) - d_model: int = 500 - n_head: int = 10 - d_head: int = 50 - d_inner: int = 1000 - div_val: int = 1 - n_layer: int = 12 - mem_len: int = 0 - clamp_len: int = -1 - same_length: bool = False - dropout: float = 0.0 - dropatt: float = 0.0 - checkpoint_activations: bool = False - offload_activations: bool = False - max_target_positions: int = II("task.max_target_positions") - - -@register_model("transformer_xl", dataclass=TransformerXLConfig) -class TransformerXLLanguageModel(FairseqLanguageModel): - @classmethod - def build_model(cls, cfg: TransformerXLConfig, task): - return cls(TransformerXLDecoder(cfg, task)) - - -class TransformerXLDecoder(FairseqIncrementalDecoder): - def __init__(self, cfg, task): - try: - from transformers.models.transfo_xl import ( - TransfoXLConfig, - TransfoXLLMHeadModel, - ) - except ImportError: - from transformers.configuration_transfo_xl import TransfoXLConfig - from transformers.modeling_transfo_xl import TransfoXLLMHeadModel - - super().__init__(task.target_dictionary) - self.cfg = cfg - - # remove any cutoffs larger than the vocab size - cutoffs = [ - cutoff for cutoff in cfg.cutoffs if cutoff < len(task.target_dictionary) - ] - - config = TransfoXLConfig( - vocab_size=len(task.target_dictionary), - cutoffs=cutoffs, - d_model=cfg.d_model, - d_embed=cfg.d_model, - n_head=cfg.n_head, - d_head=cfg.d_head, - d_inner=cfg.d_inner, - div_val=cfg.div_val, - n_layer=cfg.n_layer, - mem_len=cfg.mem_len, - clamp_len=cfg.clamp_len, - same_length=cfg.same_length, - dropout=cfg.dropout, - dropatt=cfg.dropatt, - ) - logger.info(config) - self.model = TransfoXLLMHeadModel(config) - - if cfg.checkpoint_activations or cfg.offload_activations: - for i in range(len(self.model.transformer.layers)): - self.model.transformer.layers[i] = checkpoint_wrapper( - self.model.transformer.layers[i], - offload_to_cpu=cfg.offload_activations, - ) - # TODO: may save mem to wrap(layer.pos_ff.CoreNet[3]) - - self._mems = None - - def forward( - self, - src_tokens, - src_lengths=None, # unused - incremental_state: Optional[Dict[str, List[torch.Tensor]]] = None, - encoder_out=None, - ): - if incremental_state is not None: # used during inference - mems = self.get_incremental_state(incremental_state, "mems") - src_tokens = src_tokens[:, -1:] # only keep the most recent token - else: - mems = self._mems - - output = self.model( - input_ids=src_tokens, - mems=mems, - return_dict=False, - ) - - if len(output) >= 2: - if incremental_state is not None: - self.set_incremental_state(incremental_state, "mems", output[1]) - else: - self._mems = output[1] - - return (output[0],) - - def max_positions(self): - return self.cfg.max_target_positions - - def reorder_incremental_state( - self, - incremental_state: Dict[str, Dict[str, Optional[torch.Tensor]]], - new_order: torch.Tensor, - ): - """Reorder incremental state. - - This will be called when the order of the input has changed from the - previous time step. A typical use case is beam search, where the input - order changes between time steps based on the selection of beams. - """ - mems = self.get_incremental_state(incremental_state, "mems") - if mems is not None: - new_mems = [mems_i.index_select(1, new_order) for mems_i in mems] - self.set_incremental_state(incremental_state, "mems", new_mems) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/truncated_bptt/truncated_bptt_lm_task.py b/audio/safeear/safeear_repo/fairseq_ours/examples/truncated_bptt/truncated_bptt_lm_task.py deleted file mode 100644 index e42e243d44b5451293eb1b7ce618da49524aa47a..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/truncated_bptt/truncated_bptt_lm_task.py +++ /dev/null @@ -1,284 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import logging -import os -from dataclasses import dataclass, field -from typing import List, Optional, Tuple - -import torch -from fairseq import utils -from fairseq.data import ( - Dictionary, - TokenBlockDataset, - data_utils, - iterators, -) -from fairseq.dataclass import FairseqDataclass -from fairseq.distributed import utils as dist_utils -from fairseq.tasks import FairseqTask, register_task -from omegaconf import II - -logger = logging.getLogger(__name__) - - -@dataclass -class TruncatedBPTTLMConfig(FairseqDataclass): - data: str = field(default="???", metadata={"help": "path to data directory"}) - tokens_per_sample: int = field( - default=1024, metadata={"help": "max number of tokens per sequence"}, - ) - batch_size: int = II("dataset.batch_size") - # Some models use *max_target_positions* to know how many positional - # embeddings to learn. We use II(...) to make it default to - # *tokens_per_sample*, but in principle there could be more positional - # embeddings than tokens in a single batch. This may also be irrelevant for - # custom model implementations. - max_target_positions: int = II("task.tokens_per_sample") - # these will be populated automatically if not provided - data_parallel_rank: Optional[int] = None - data_parallel_size: Optional[int] = None - - -@register_task("truncated_bptt_lm", dataclass=TruncatedBPTTLMConfig) -class TruncatedBPTTLMTask(FairseqTask): - def __init__(self, cfg: TruncatedBPTTLMConfig): - super().__init__(cfg) - - if cfg.data_parallel_rank is None or cfg.data_parallel_size is None: - if torch.distributed.is_initialized(): - cfg.data_parallel_rank = dist_utils.get_data_parallel_rank() - cfg.data_parallel_size = dist_utils.get_data_parallel_world_size() - else: - cfg.data_parallel_rank = 0 - cfg.data_parallel_size = 1 - - # load the dictionary - paths = utils.split_paths(cfg.data) - assert len(paths) > 0 - self.dictionary = Dictionary.load(os.path.join(paths[0], "dict.txt")) - logger.info("dictionary: {} types".format(len(self.dictionary))) - - def load_dataset(self, split, epoch=1, combine=False, **kwargs): - """Load a given dataset split (e.g., train, valid, test)""" - - # support sharded datasets - paths = utils.split_paths(self.cfg.data) - assert len(paths) > 0 - data_path = paths[(epoch - 1) % len(paths)] - split_path = os.path.join(data_path, split) - - # each element of *data* will be a tensorized line from the original - # text dataset, similar to ``open(split_path).readlines()`` - data = data_utils.load_indexed_dataset( - split_path, self.dictionary, combine=combine - ) - if data is None: - raise FileNotFoundError( - "Dataset not found: {} ({})".format(split, split_path) - ) - - # this is similar to ``data.view(-1).split(tokens_per_sample)`` - data = TokenBlockDataset( - data, - data.sizes, - block_size=self.cfg.tokens_per_sample, - pad=None, # unused - eos=None, # unused - break_mode="none", - ) - - self.datasets[split] = TruncatedBPTTDataset( - data=data, - bsz_per_shard=self.cfg.batch_size, - shard_id=self.cfg.data_parallel_rank, - num_shards=self.cfg.data_parallel_size, - ) - - def dataset(self, split): - return self.datasets[split] - - def get_batch_iterator( - self, - dataset, - num_workers=0, - epoch=1, - data_buffer_size=0, - skip_remainder_batch=False, - **kwargs - ): - return iterators.EpochBatchIterator( - dataset=dataset, - collate_fn=self._collate_fn, - num_workers=num_workers, - epoch=epoch, - buffer_size=data_buffer_size, - # we don't use the batching functionality from EpochBatchIterator; - # instead every item in *dataset* is a whole batch - batch_sampler=[[i] for i in range(len(dataset))], - disable_shuffling=True, - skip_remainder_batch=skip_remainder_batch, - ) - - def _collate_fn(self, items: List[List[torch.Tensor]]): - # we don't use fairseq's batching functionality, so we expect a single - # Tensor of type List[torch.Tensor] - assert len(items) == 1 - - # item will have shape B x T (the last batch may have length < T) - id, item = items[0] - item = data_utils.collate_tokens(item, pad_idx=self.source_dictionary.pad()) - B, T = item.size() - - # shift item one position over and append a padding token for the target - target = torch.nn.functional.pad( - item[:, 1:], (0, 1, 0, 0), value=self.target_dictionary.pad() - ) - - # fairseq expects batches to have the following structure - return { - "id": torch.tensor([id] * item.size(0)), - "net_input": {"src_tokens": item,}, - "target": target, - "nsentences": item.size(0), - "ntokens": item.numel(), - } - - def build_dataset_for_inference( - self, src_tokens: List[torch.Tensor], src_lengths: List[int], **kwargs - ) -> torch.utils.data.Dataset: - eos = self.source_dictionary.eos() - dataset = TokenBlockDataset( - src_tokens, - src_lengths, - block_size=None, # ignored for "eos" break mode - pad=self.source_dictionary.pad(), - eos=eos, - break_mode="eos", - ) - - class Dataset(torch.utils.data.Dataset): - def __getitem__(self, i): - item = dataset[i] - if item[-1] == eos: - # remove eos to support generating with a prefix - item = item[:-1] - return (i, [item]) - - def __len__(self): - return len(dataset) - - return Dataset() - - def inference_step( - self, generator, models, sample, prefix_tokens=None, constraints=None - ): - with torch.no_grad(): - if constraints is not None: - raise NotImplementedError - - # SequenceGenerator doesn't use *src_tokens* directly, we need to - # pass the *prefix_tokens* argument instead. - if prefix_tokens is None and sample["net_input"]["src_tokens"].nelement(): - prefix_tokens = sample["net_input"]["src_tokens"] - - # begin generation with the end-of-sentence token - bos_token = self.source_dictionary.eos() - - return generator.generate( - models, sample, prefix_tokens=prefix_tokens, bos_token=bos_token - ) - - def eval_lm_dataloader( - self, - dataset, - max_tokens: Optional[int] = 36000, - batch_size: Optional[int] = None, - max_positions: Optional[int] = None, - num_shards: int = 1, - shard_id: int = 0, - num_workers: int = 1, - data_buffer_size: int = 10, - context_window: int = 0, - ): - if context_window > 0: - raise NotImplementedError( - "Transformer-XL doesn't need --context-window, try " - "--model-overrides '{\"mem_len\":42}' instead " - ) - return self.get_batch_iterator( - dataset=dataset, - max_tokens=max_tokens, - max_sentences=batch_size, - max_positions=max_positions, - ignore_invalid_inputs=True, - num_shards=num_shards, - shard_id=shard_id, - num_workers=num_workers, - data_buffer_size=data_buffer_size, - ).next_epoch_itr(shuffle=False) - - @property - def source_dictionary(self): - return self.dictionary - - @property - def target_dictionary(self): - return self.dictionary - - -class TruncatedBPTTDataset(torch.utils.data.Dataset): - def __init__( - self, - data: List[torch.Tensor], # ordered list of items - bsz_per_shard, # number of items processed per GPUs per forward - shard_id, # current GPU ID - num_shards, # number of GPUs - ): - super().__init__() - self.data = data - - def batchify(data, bsz): - # Work out how cleanly we can divide the dataset into bsz parts. - nbatch = data.size(0) // bsz - # Trim off any extra elements that wouldn't cleanly fit (remainders). - data = data.narrow(0, 0, nbatch * bsz) - # Evenly divide the data across the bsz batches. - data = data.view(bsz, -1).contiguous() - return data - - # total number of sequences processed by all GPUs in each forward pass - global_batch_size = bsz_per_shard * num_shards - - """ - With a 16 item dataset, bsz_per_shard=2 and num_shards=3, - *indices* might look like: - - indices = [[0, 1], - [2, 3], - [4, 5], - [6, 7], - [8, 9], - [10, 11]] - - The size of the TruncatedBPTTDataset instance will be 2, - and shard 1 will see items: - - [(0, [data[4], data[6]]), - (1, [data[5], data[7]])] - """ - indices = batchify(torch.arange(len(data)), global_batch_size) - assert indices.size(0) == global_batch_size - - self.my_indices = indices[ - shard_id * bsz_per_shard : (shard_id + 1) * bsz_per_shard - ] - assert self.my_indices.size(0) == bsz_per_shard - - def __len__(self): - return self.my_indices.size(1) - - def __getitem__(self, i) -> Tuple[int, List[torch.Tensor]]: - return (i, [self.data[idx] for idx in self.my_indices[:, i]]) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/unsupervised_quality_estimation/README.md b/audio/safeear/safeear_repo/fairseq_ours/examples/unsupervised_quality_estimation/README.md deleted file mode 100644 index e86a0d13b883af0c37fdc2c1fee9b0b9dff4d18c..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/unsupervised_quality_estimation/README.md +++ /dev/null @@ -1,126 +0,0 @@ -# Unsupervised Quality Estimation for Neural Machine Translation (Fomicheva et al., 2020) - -This page includes instructions for reproducing results from the paper [Unsupervised Quality Estimation for Neural -Machine Translation (Fomicheva et al., 2020)](https://arxiv.org/abs/2005.10608) - -## Requirements: - -* mosesdecoder: https://github.com/moses-smt/mosesdecoder -* subword-nmt: https://github.com/rsennrich/subword-nmt -* flores: https://github.com/facebookresearch/flores - -## Download Models and Test Data - -Download translation models and test data from [MLQE dataset repository](https://github.com/facebookresearch/mlqe). - -## Set up: - -Given a testset consisting of source sentences and reference translations: - -* `SRC_LANG`: source language -* `TGT_LANG`: target language -* `INPUT`: input prefix, such that the file `$INPUT.$SRC_LANG` contains source sentences and `$INPUT.$TGT_LANG` -contains the reference sentences -* `OUTPUT_DIR`: output path to store results -* `MOSES_DECODER`: path to mosesdecoder installation -* `BPE_ROOT`: path to subword-nmt installation -* `BPE`: path to BPE model -* `MODEL_DIR`: directory containing the NMT model `.pt` file as well as the source and target vocabularies. -* `TMP`: directory for intermediate temporary files -* `GPU`: if translating with GPU, id of the GPU to use for inference -* `DROPOUT_N`: number of stochastic forward passes - -`$DROPOUT_N` is set to 30 in the experiments reported in the paper. However, we observed that increasing it beyond 10 -does not bring substantial improvements. - -## Translate the data using standard decoding - -Preprocess the input data: -``` -for LANG in $SRC_LANG $TGT_LANG; do - perl $MOSES_DECODER/scripts/tokenizer/tokenizer.perl -threads 80 -a -l $LANG < $INPUT.$LANG > $TMP/preprocessed.tok.$LANG - python $BPE_ROOT/apply_bpe.py -c ${BPE} < $TMP/preprocessed.tok.$LANG > $TMP/preprocessed.tok.bpe.$LANG -done -``` - -Binarize the data for faster translation: - -``` -fairseq-preprocess --srcdict $MODEL_DIR/dict.$SRC_LANG.txt --tgtdict $MODEL_DIR/dict.$TGT_LANG.txt ---source-lang ${SRC_LANG} --target-lang ${TGT_LANG} --testpref $TMP/preprocessed.tok.bpe --destdir $TMP/bin --workers 4 -``` - -Translate - -``` -CUDA_VISIBLE_DEVICES=$GPU fairseq-generate $TMP/bin --path ${MODEL_DIR}/${SRC_LANG}-${TGT_LANG}.pt --beam 5 ---source-lang $SRC_LANG --target-lang $TGT_LANG --no-progress-bar --unkpen 5 > $TMP/fairseq.out -grep ^H $TMP/fairseq.out | cut -d- -f2- | sort -n | cut -f3- > $TMP/mt.out -``` - -Post-process - -``` -sed -r 's/(@@ )| (@@ ?$)//g' < $TMP/mt.out | perl $MOSES_DECODER/scripts/tokenizer/detokenizer.perl --l $TGT_LANG > $OUTPUT_DIR/mt.out -``` - -## Produce uncertainty estimates - -### Scoring - -Make temporary files to store the translations repeated N times. - -``` -python ${SCRIPTS}/scripts/uncertainty/repeat_lines.py -i $TMP/preprocessed.tok.bpe.$SRC_LANG -n $DROPOUT_N --o $TMP/repeated.$SRC_LANG -python ${SCRIPTS}/scripts/uncertainty/repeat_lines.py -i $TMP/mt.out -n $DROPOUT_N -o $TMP/repeated.$TGT_LANG - -fairseq-preprocess --srcdict ${MODEL_DIR}/dict.${SRC_LANG}.txt $TGT_DIC --source-lang ${SRC_LANG} ---target-lang ${TGT_LANG} --testpref ${TMP}/repeated --destdir ${TMP}/bin-repeated -``` - -Produce model scores for the generated translations using `--retain-dropout` option to apply dropout at inference time: - -``` -CUDA_VISIBLE_DEVICES=${GPU} fairseq-generate ${TMP}/bin-repeated --path ${MODEL_DIR}/${LP}.pt --beam 5 - --source-lang $SRC_LANG --target-lang $TGT_LANG --no-progress-bar --unkpen 5 --score-reference --retain-dropout - --retain-dropout-modules '["TransformerModel","TransformerEncoder","TransformerDecoder","TransformerEncoderLayer"]' - TransformerDecoderLayer --seed 46 > $TMP/dropout.scoring.out - -grep ^H $TMP/dropout.scoring.out | cut -d- -f2- | sort -n | cut -f2 > $TMP/dropout.scores - -``` - -Use `--retain-dropout-modules` to specify the modules. By default, dropout is applied in the same places -as for training. - -Compute the mean of the resulting output distribution: - -``` -python $SCRIPTS/scripts/uncertainty/aggregate_scores.py -i $TMP/dropout.scores -o $OUTPUT_DIR/dropout.scores.mean --n $DROPOUT_N -``` - -### Generation - -Produce multiple translation hypotheses for the same source using `--retain-dropout` option: - -``` -CUDA_VISIBLE_DEVICES=${GPU} fairseq-generate ${TMP}/bin-repeated --path ${MODEL_DIR}/${LP}.pt - --beam 5 --source-lang $SRC_LANG --target-lang $TGT_LANG --no-progress-bar --retain-dropout - --unkpen 5 --retain-dropout-modules TransformerModel TransformerEncoder TransformerDecoder -TransformerEncoderLayer TransformerDecoderLayer --seed 46 > $TMP/dropout.generation.out - -grep ^H $TMP/dropout.generation.out | cut -d- -f2- | sort -n | cut -f3- > $TMP/dropout.hypotheses_ - -sed -r 's/(@@ )| (@@ ?$)//g' < $TMP/dropout.hypotheses_ | perl $MOSES_DECODER/scripts/tokenizer/detokenizer.perl --l $TGT_LANG > $TMP/dropout.hypotheses -``` - -Compute similarity between multiple hypotheses corresponding to the same source sentence using Meteor -evaluation metric: -``` -python meteor.py -i $TMP/dropout.hypotheses -m <path_to_meteor_installation> -n $DROPOUT_N -o -$OUTPUT_DIR/dropout.gen.sim.meteor -``` diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/unsupervised_quality_estimation/aggregate_scores.py b/audio/safeear/safeear_repo/fairseq_ours/examples/unsupervised_quality_estimation/aggregate_scores.py deleted file mode 100644 index 713099fe45cb57838f6ff5cc80b302cbff3df182..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/unsupervised_quality_estimation/aggregate_scores.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import sys - -import numpy as np - -aggregate_funcs = { - "std": np.std, - "var": np.var, - "median": np.median, - "mean": np.mean, - "min": np.min, - "max": np.max, -} - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("-i", "--input_file", required=True, type=str) - parser.add_argument("-n", "--repeat_times", required=True, type=int) - parser.add_argument("-o", "--output_file", required=False) - parser.add_argument("-f", "--func", required=False, default="mean") - args = parser.parse_args() - - stream = open(args.output_file, "w") if args.output_file else sys.stdout - - segment_scores = [] - for line in open(args.input_file): - segment_scores.append(float(line.strip())) - if len(segment_scores) == args.repeat_times: - stream.write("{}\n".format(aggregate_funcs[args.func](segment_scores))) - segment_scores = [] - - -if __name__ == "__main__": - main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/unsupervised_quality_estimation/meteor.py b/audio/safeear/safeear_repo/fairseq_ours/examples/unsupervised_quality_estimation/meteor.py deleted file mode 100644 index 2ee0448cf1f167f6f3ecee56ad807922cffb0956..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/unsupervised_quality_estimation/meteor.py +++ /dev/null @@ -1,109 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import math -import os -import subprocess -import sys -import tempfile -from collections import defaultdict -from itertools import combinations - - -def read_translations(path, n_repeats): - segment_counter = 0 - segment_translations = [] - translations = defaultdict(list) - for line in open(path): - segment_translations.append(" ".join(line.split())) - if len(segment_translations) == n_repeats: - translations[segment_counter] = segment_translations - segment_translations = [] - segment_counter += 1 - return translations - - -def generate_input(translations, n_repeats): - _, ref_path = tempfile.mkstemp() - _, mt_path = tempfile.mkstemp() - ref_fh = open(ref_path, "w") - mt_fh = open(mt_path, "w") - for segid in sorted(translations.keys()): - assert len(translations[segid]) == n_repeats - indexes = combinations(range(n_repeats), 2) - for idx1, idx2 in indexes: - mt_fh.write(translations[segid][idx1].strip() + "\n") - ref_fh.write(translations[segid][idx2].strip() + "\n") - sys.stderr.write("\nSaved translations to %s and %s" % (ref_path, mt_path)) - return ref_path, mt_path - - -def run_meteor(ref_path, mt_path, metric_path, lang="en"): - _, out_path = tempfile.mkstemp() - subprocess.call( - [ - "java", - "-Xmx2G", - "-jar", - metric_path, - mt_path, - ref_path, - "-p", - "0.5 0.2 0.6 0.75", # default parameters, only changed alpha to give equal weight to P and R - "-norm", - "-l", - lang, - ], - stdout=open(out_path, "w"), - ) - os.remove(ref_path) - os.remove(mt_path) - sys.stderr.write("\nSaved Meteor output to %s" % out_path) - return out_path - - -def read_output(meteor_output_path, n_repeats): - n_combinations = math.factorial(n_repeats) / ( - math.factorial(2) * math.factorial(n_repeats - 2) - ) - raw_scores = [] - average_scores = [] - for line in open(meteor_output_path): - if not line.startswith("Segment "): - continue - score = float(line.strip().split("\t")[1]) - raw_scores.append(score) - if len(raw_scores) == n_combinations: - average_scores.append(sum(raw_scores) / n_combinations) - raw_scores = [] - os.remove(meteor_output_path) - return average_scores - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("-i", "--infile") - parser.add_argument("-n", "--repeat_times", type=int) - parser.add_argument("-m", "--meteor") - parser.add_argument("-o", "--output") - args = parser.parse_args() - - translations = read_translations(args.infile, args.repeat_times) - sys.stderr.write("\nGenerating input for Meteor...") - ref_path, mt_path = generate_input(translations, args.repeat_times) - sys.stderr.write("\nRunning Meteor...") - out_path = run_meteor(ref_path, mt_path, args.meteor) - sys.stderr.write("\nReading output...") - scores = read_output(out_path, args.repeat_times) - sys.stderr.write("\nWriting results...") - with open(args.output, "w") as o: - for scr in scores: - o.write("{}\n".format(scr)) - o.close() - - -if __name__ == "__main__": - main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/unsupervised_quality_estimation/repeat_lines.py b/audio/safeear/safeear_repo/fairseq_ours/examples/unsupervised_quality_estimation/repeat_lines.py deleted file mode 100644 index 5a04851a74624e9c8ebc259805b7aed6c638b0de..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/unsupervised_quality_estimation/repeat_lines.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import sys - - -def _normalize_spaces(line): - return " ".join(line.split()) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("-i", "--input_file", required=True, type=str) - parser.add_argument("-n", "--repeat_times", required=True, type=int) - parser.add_argument("-o", "--output_file", required=False, type=str) - args = parser.parse_args() - stream = open(args.output_file, "w") if args.output_file else sys.stdout - - for line in open(args.input_file): - for _ in range(args.repeat_times): - stream.write(_normalize_spaces(line) + "\n") - - -if __name__ == "__main__": - main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/README.md b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/README.md deleted file mode 100644 index e97973307505e902307aa24b3885800378193b5d..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/README.md +++ /dev/null @@ -1,426 +0,0 @@ -# wav2vec 2.0 - -wav2vec 2.0 learns speech representations on unlabeled data as described in [wav2vec 2.0: A Framework for Self-Supervised Learning of Speech Representations (Baevski et al., 2020)](https://arxiv.org/abs/2006.11477). - -We learned speech representations in multiple languages as well in [Unsupervised Cross-lingual Representation Learning for Speech Recognition (Conneau et al., 2020)](https://arxiv.org/abs/2006.13979). - -We also combined wav2vec 2.0 with self-training in [Self-training and Pre-training are Complementary for Speech Recognition (Xu et al., 2020)](https://arxiv.org/abs/2010.11430). - -We combined speech data from multiple domains in [Robust wav2vec 2.0: Analyzing Domain Shift in Self-Supervised Pre-Training (Hsu, et al., 2021)](https://arxiv.org/abs/2104.01027). - -We finetuned XLSR-53 on multiple languages to transcribe unseen languages in [Simple and Effective Zero-shot Cross-lingual Phoneme Recognition (Xu et al., 2021)](https://arxiv.org/abs/2109.11680). - -## Pre-trained models - -Model | Finetuning split | Dataset | Model -|---|---|---|--- -Wav2Vec 2.0 Base | No finetuning | [Librispeech](http://www.openslr.org/12) | [download](https://dl.fbaipublicfiles.com/fairseq/wav2vec/wav2vec_small.pt) -Wav2Vec 2.0 Base | 10 minutes | [Librispeech](http://www.openslr.org/12) | [download](https://dl.fbaipublicfiles.com/fairseq/wav2vec/wav2vec_small_10m.pt) -Wav2Vec 2.0 Base | 100 hours | [Librispeech](http://www.openslr.org/12) | [download](https://dl.fbaipublicfiles.com/fairseq/wav2vec/wav2vec_small_100h.pt) -Wav2Vec 2.0 Base | 960 hours | [Librispeech](http://www.openslr.org/12) | [download](https://dl.fbaipublicfiles.com/fairseq/wav2vec/wav2vec_small_960h.pt) -Wav2Vec 2.0 Large | No finetuning | [Librispeech](http://www.openslr.org/12) | [download](https://dl.fbaipublicfiles.com/fairseq/wav2vec/libri960_big.pt) -Wav2Vec 2.0 Large | 10 minutes | [Librispeech](http://www.openslr.org/12) | [download](https://dl.fbaipublicfiles.com/fairseq/wav2vec/wav2vec_big_10m.pt) -Wav2Vec 2.0 Large | 100 hours | [Librispeech](http://www.openslr.org/12) | [download](https://dl.fbaipublicfiles.com/fairseq/wav2vec/wav2vec_big_100h.pt) -Wav2Vec 2.0 Large | 960 hours | [Librispeech](http://www.openslr.org/12) | [download](https://dl.fbaipublicfiles.com/fairseq/wav2vec/wav2vec_big_960h.pt) -Wav2Vec 2.0 Large (LV-60)* | No finetuning | [Libri-Light](https://github.com/facebookresearch/libri-light) | [download](https://dl.fbaipublicfiles.com/fairseq/wav2vec/wav2vec_vox_new.pt) -Wav2Vec 2.0 Large conformer - rel_pos (LV-60)* | No finetuning | [Libri-Light](https://github.com/facebookresearch/libri-light) | [download](s3://dl.fbaipublicfiles.com/fairseq/conformer/wav2vec2/librilight/LL_relpos_PT_no_FT) -Wav2Vec 2.0 Large conformer - rope (LV-60)* | No finetuning | [Libri-Light](https://github.com/facebookresearch/libri-light) | [download](s3://dl.fbaipublicfiles.com/fairseq/conformer/wav2vec2/librilight/LL_rope_PT_no_FT) -Wav2Vec 2.0 Large (LV-60)* | 10 minutes | [Libri-Light](https://github.com/facebookresearch/libri-light) + [Librispeech](http://www.openslr.org/12) | [download](https://dl.fbaipublicfiles.com/fairseq/wav2vec/wav2vec_vox_10m_new.pt) -Wav2Vec 2.0 Large (LV-60)* | 100 hours | [Libri-Light](https://github.com/facebookresearch/libri-light) + [Librispeech](http://www.openslr.org/12) | [download](https://dl.fbaipublicfiles.com/fairseq/wav2vec/wav2vec_vox_100h_new.pt) -Wav2Vec 2.0 Large conformer - rel_pos (LV-60)* | 100 hours | [Libri-Light](https://github.com/facebookresearch/libri-light) | [download](s3://dl.fbaipublicfiles.com/fairseq/conformer/wav2vec2/librilight/LL_relpos_PT_100h_FT.pt) -Wav2Vec 2.0 Large conformer - rope (LV-60)* | 100 hours | [Libri-Light](https://github.com/facebookresearch/libri-light) | [download](s3://dl.fbaipublicfiles.com/fairseq/conformer/wav2vec2/librilight/LL_rope_PT_100h_FT.pt) -Wav2Vec 2.0 Large (LV-60)* | 960 hours | [Libri-Light](https://github.com/facebookresearch/libri-light) + [Librispeech](http://www.openslr.org/12) | [download](https://dl.fbaipublicfiles.com/fairseq/wav2vec/wav2vec2_vox_960h_new.pt) -Wav2Vec 2.0 Large conformer - rel_pos (LV-60)* | 960 hours | [Libri-Light](https://github.com/facebookresearch/libri-light) | [download](s3://dl.fbaipublicfiles.com/fairseq/conformer/wav2vec2/librilight/LL_relpos_PT_960h_FT.pt) -Wav2Vec 2.0 Large conformer - rope (LV-60)* | 960 hours | [Libri-Light](https://github.com/facebookresearch/libri-light) | [download](s3://dl.fbaipublicfiles.com/fairseq/conformer/wav2vec2/librilight/LL_rope_PT_960h_FT.pt) -Wav2Vec 2.0 Large (LV-60) + Self Training * | 10 minutes | [Libri-Light](https://github.com/facebookresearch/libri-light) + [Librispeech](http://www.openslr.org/12) | [download](https://dl.fbaipublicfiles.com/fairseq/wav2vec/wav2vec_vox_10m_pl.pt) -Wav2Vec 2.0 Large (LV-60) + Self Training * | 100 hours | [Libri-Light](https://github.com/facebookresearch/libri-light) + [Librispeech](http://www.openslr.org/12) | [download](https://dl.fbaipublicfiles.com/fairseq/wav2vec/wav2vec_vox_100h_pl.pt) -Wav2Vec 2.0 Large (LV-60) + Self Training * | 960 hours | [Libri-Light](https://github.com/facebookresearch/libri-light) + [Librispeech](http://www.openslr.org/12) | [download](https://dl.fbaipublicfiles.com/fairseq/wav2vec/wav2vec_vox_960h_pl.pt) -Wav2Vec 2.0 Large (LV-60 + CV + SWBD + FSH) ** | No finetuning | [Libri-Light](https://github.com/facebookresearch/libri-light) + [CommonVoice](https://commonvoice.mozilla.org/en/languages) + [Switchboard](https://catalog.ldc.upenn.edu/LDC97S62) + [Fisher](https://catalog.ldc.upenn.edu/LDC2004T19) | [download](https://dl.fbaipublicfiles.com/fairseq/wav2vec/w2v_large_lv_fsh_swbd_cv.pt) -Wav2Vec 2.0 Large (LV-60 + CV + SWBD + FSH) ** | 960 hours Librispeech | [Libri-Light](https://github.com/facebookresearch/libri-light) + [CommonVoice](https://commonvoice.mozilla.org/en/languages) + [Switchboard](https://catalog.ldc.upenn.edu/LDC97S62) + [Fisher](https://catalog.ldc.upenn.edu/LDC2004T19) | [download](https://dl.fbaipublicfiles.com/fairseq/wav2vec/w2v_large_lv_fsh_swbd_cv_ftls960_updated.pt) -Wav2Vec 2.0 Large (LV-60 + CV + SWBD + FSH) ** | 300 hours Switchboard | [Libri-Light](https://github.com/facebookresearch/libri-light) + [CommonVoice](https://commonvoice.mozilla.org/en/languages) + [Switchboard](https://catalog.ldc.upenn.edu/LDC97S62) + [Fisher](https://catalog.ldc.upenn.edu/LDC2004T19) | [download](https://dl.fbaipublicfiles.com/fairseq/wav2vec/w2v_large_lv_fsh_swbd_cv_ftsb300_updated.pt) - -\* updated (Oct. 24, 2020)\ -** updated (Nov. 13, 2021) - -We also release multilingual pre-trained wav2vec 2.0 (XLSR) models: - -Model | Architecture | Hours | Languages | Datasets | Model -|---|---|---|---|---|--- -XLSR-53 | Large | 56k | 53 | MLS, CommonVoice, BABEL | [download](https://dl.fbaipublicfiles.com/fairseq/wav2vec/xlsr_53_56k.pt) - -The XLSR model uses the following datasets for multilingual pretraining: - -* **[MLS: Multilingual LibriSpeech](https://indico2.conference4me.psnc.pl/event/35/contributions/3585/attachments/1060/1101/Wed-2-6-10.pdf)** (8 languages, 50.7k hours): *Dutch, English, French, German, Italian, Polish, Portuguese, Spanish* - -* **[CommonVoice](https://commonvoice.mozilla.org/en/languages)** (36 languages, 3.6k hours): *Arabic, Basque, Breton, Chinese (CN), Chinese (HK), Chinese (TW), Chuvash, Dhivehi, Dutch, English, Esperanto, Estonian, French, German, Hakh-Chin, Indonesian, Interlingua, Irish, Italian, Japanese, Kabyle, Kinyarwanda, Kyrgyz, Latvian, Mongolian, Persian, Portuguese, Russian, Sakha, Slovenian, Spanish, Swedish, Tamil, Tatar, Turkish, Welsh* (see also [finetuning splits]([https://dl.fbaipublicfiles.com/cpc_audio/common_voices_splits.tar.gz]) from [this paper](https://arxiv.org/abs/2002.02848)). - -* **[Babel](https://catalog.ldc.upenn.edu/byyear)** (17 languages, 1.7k hours): *Assamese, Bengali, Cantonese, Cebuano, Georgian, Haitian, Kazakh, Kurmanji, Lao, Pashto, Swahili, Tagalog, Tamil, Tok, Turkish, Vietnamese, Zulu* - -We also finetuned several models on languages from [CommonVoice](https://commonvoice.mozilla.org/en/languages) (version 6.1) and [Babel](https://catalog.ldc.upenn.edu/byyear). Please refer to [our paper](https://arxiv.org/abs/2109.11680) for details about which languages are used. - -Pretrained Model | Fintune Dataset | # Languages | Phonemizer | Model | Dictionary -|---|---|---|---|---|--- -LV-60 | CommonVoice | 26 | [Espeak](https://github.com/espeak-ng/espeak-ng/blob/master/docs/languages.md) | [download](https://dl.fbaipublicfiles.com/fairseq/wav2vec/zero_shot/espeak_en_26lang_m10.pt) | [download](https://dl.fbaipublicfiles.com/fairseq/wav2vec/zero_shot/espeak_dict.txt) -XLSR-53 | CommonVoice | 26 | [Espeak](https://github.com/espeak-ng/espeak-ng/blob/master/docs/languages.md) | [download](https://dl.fbaipublicfiles.com/fairseq/wav2vec/zero_shot/espeak_26lang_m10.pt) | [download](https://dl.fbaipublicfiles.com/fairseq/wav2vec/zero_shot/espeak_dict.txt) -XLSR-53 | CommonVoice | 21 | [Phonetisaurus](https://github.com/AdolfVonKleist/Phonetisaurus) | [download](https://dl.fbaipublicfiles.com/fairseq/wav2vec/zero_shot/phonetisaurus_21lang_m10.pt) | [download](https://dl.fbaipublicfiles.com/fairseq/wav2vec/zero_shot/phonetisaurus_dict.txt) -XLSR-53 | CommonVoice, BABEL | 21, 19 | [Phonetisaurus](https://github.com/AdolfVonKleist/Phonetisaurus) | [download](https://dl.fbaipublicfiles.com/fairseq/wav2vec/zero_shot/phonetisaurus_40lang_m10.pt) | [download](https://dl.fbaipublicfiles.com/fairseq/wav2vec/zero_shot/phonetisaurus_40lang.dict.txt) - -We release 2 models that are finetuned on data from 2 different phonemizers. Although the phonemes are all [IPA](https://en.wikipedia.org/wiki/International_Phonetic_Alphabet) symbols, there are still subtle differences between the phonemized transcriptions from the 2 phonemizers. Thus, it's better to use the corresponding model, if your data is phonemized by either phonemizer above. - -## Training a new model with the CLI tools - -Given a directory containing wav files to be used for pretraining (we recommend splitting each file into separate file 10 to 30 seconds in length) - -### Prepare training data manifest - -First, install the `soundfile` library: - -```shell script -pip install soundfile -``` - -Next, run: - -```shell script -python examples/wav2vec/wav2vec_manifest.py /path/to/waves --dest /manifest/path --ext $ext --valid-percent $valid -``` - -$ext should be set to flac, wav, or whatever format your dataset happens to use that soundfile can read. - -$valid should be set to some reasonable percentage (like 0.01) of training data to use for validation. -To use a pre-defined validation set (like dev-other from librispeech), set to it 0 and then overwrite valid.tsv with a -separately pre-processed manifest file. - -### Train a wav2vec 2.0 base model - -This configuration was used for the base model trained on the Librispeech dataset in the wav2vec 2.0 paper - -Note that the input is expected to be single channel, sampled at 16 kHz - -```shell script -$ fairseq-hydra-train \ - task.data=/path/to/data \ - --config-dir /path/to/fairseq-py/examples/wav2vec/config/pretraining \ - --config-name wav2vec2_base_librispeech -``` - -Note: you can simulate 64 GPUs by using k GPUs and adding command line parameters (before `--config-dir`) -`distributed_training.distributed_world_size=k` `+optimization.update_freq='[x]'` where x = 64/k - -### Train a wav2vec 2.0 large model - -This configuration was used for the large model trained on the Libri-light dataset in the wav2vec 2.0 paper - -```shell script -$ fairseq-hydra-train \ - task.data=/path/to/data \ - --config-dir /path/to/fairseq-py/examples/wav2vec/config/pretraining \ - --config-name wav2vec2_large_librivox -``` - -Note: you can simulate 128 GPUs by using k GPUs and adding command line parameters (before `--config-dir`) -`distributed_training.distributed_world_size=k` `+optimization.update_freq='[x]'` where x = 128/k - -### Train a wav2vec 2.0 model with conformer backbone - -To replace the transformer layers in the encoder with the conformer layers, set `--layer-type conformer --attn-type espnet --pos-enc-type ${POS_ENC_TYPE}`. `POS_ENC_TYPE` refers to positional encoding to be used in the conformer encoder. -Set it to `abs`, `rope` or `rel_pos` to use the absolute positional encoding, rotary positional encoding or relative positional encoding in the conformer layer respectively. - -To train a base model with conformer: - -```shell script -$ fairseq-hydra-train \ - task.data=/path/to/data \ - --config-dir /path/to/fairseq-py/examples/wav2vec/config/pretraining \ - --config-name wav2vec2_conformer_base_librispeech \ - --attn-type espnet --pos-enc-type ${POS_ENC_TYPE} -``` - -To train a large model with conformer: - -```shell script -$ fairseq-hydra-train \ - task.data=/path/to/data \ - --config-dir /path/to/fairseq-py/examples/wav2vec/config/pretraining \ - --config-name wav2vec2_conformer_large_librivox - --attn-type espnet --pos-enc-type ${POS_ENC_TYPE} - -``` - -### Fine-tune a pre-trained model with CTC - -Fine-tuning a model requires parallel audio and labels file, as well as a vocabulary file in fairseq format. -A letter vocabulary can be downloaded [here](https://dl.fbaipublicfiles.com/fairseq/wav2vec/dict.ltr.txt). -An example [script](libri_labels.py) that generates labels for the Librispeech dataset from the tsv file produced by wav2vec_manifest.py can be used as follows: - -```shell script -split=train -$ python libri_labels.py /path/to/tsv --output-dir /output/dir --output-name $split -``` - -Fine-tuning on 100h of Librispeech with letter targets: - -```shell script -$ fairseq-hydra-train \ - distributed_training.distributed_port=$PORT \ - task.data=/path/to/data \ - model.w2v_path=/path/to/model.pt \ - --config-dir /path/to/fairseq-py/examples/wav2vec/config/finetuning \ - --config-name base_100h -``` - -There are other config files in the config/finetuning directory that can be used to fine-tune on other splits. -You can specify the right config via the `--config-name` parameter. - -Note: you can simulate 24 GPUs by using k GPUs and adding command line parameters (before `--config-dir`) -`distributed_training.distributed_world_size=k` `+optimization.update_freq='[x]'` where x = 24/k - -Decoding with a language model during training requires flashlight [python bindings](https://github.com/facebookresearch/flashlight/tree/master/bindings/python) (previously called [wav2letter](https://github.com/facebookresearch/wav2letter). -If you want to use a language model, add `+criterion.wer_args='[/path/to/kenlm, /path/to/lexicon, 2, -1]'` to the command line. - -### Evaluating a CTC model - -Evaluating a CTC model with a language model requires [flashlight python bindings](https://github.com/facebookresearch/flashlight/tree/master/bindings/python) (previously called [wav2letter](https://github.com/facebookresearch/wav2letter) to be installed. - -Fairseq transformer language model used in the wav2vec 2.0 paper can be obtained from the [wav2letter model repository](https://github.com/facebookresearch/wav2letter/tree/master/recipes/sota/2019). -Be sure to upper-case the language model vocab after downloading it. - -Letter dictionary for pre-trained models can be found [here](https://dl.fbaipublicfiles.com/fairseq/wav2vec/dict.ltr.txt). - -Next, run the evaluation command: - -```shell script -$subset=dev_other -python examples/speech_recognition/infer.py /checkpoint/abaevski/data/speech/libri/10h/wav2vec/raw --task audio_finetuning \ ---nbest 1 --path /path/to/model --gen-subset $subset --results-path /path/to/save/results/for/sclite --w2l-decoder kenlm \ ---lm-model /path/to/kenlm.bin --lm-weight 2 --word-score -1 --sil-weight 0 --criterion ctc --labels ltr --max-tokens 4000000 \ ---post-process letter -``` - -To get raw numbers, use --w2l-decoder viterbi and omit the lexicon. To use the transformer language model, use --w2l-decoder fairseqlm. - -## Use wav2vec 2.0 with 🤗Transformers - -Wav2Vec2 is also available in the [🤗Transformers library](https://github.com/huggingface/transformers) since version 4.4. - -Pretrained Models can be found on the [hub](https://huggingface.co/models?filter=wav2vec2) -and documentation can be found [here](https://huggingface.co/transformers/master/model_doc/wav2vec2.html). - -Usage example: - -```python -# !pip install transformers -# !pip install datasets -import soundfile as sf -import torch -from datasets import load_dataset -from transformers import Wav2Vec2ForCTC, Wav2Vec2Processor - -# load pretrained model -processor = Wav2Vec2Processor.from_pretrained("facebook/wav2vec2-base-960h") -model = Wav2Vec2ForCTC.from_pretrained("facebook/wav2vec2-base-960h") - - -librispeech_samples_ds = load_dataset("patrickvonplaten/librispeech_asr_dummy", "clean", split="validation") - -# load audio -audio_input, sample_rate = sf.read(librispeech_samples_ds[0]["file"]) - -# pad input values and return pt tensor -input_values = processor(audio_input, sampling_rate=sample_rate, return_tensors="pt").input_values - -# INFERENCE - -# retrieve logits & take argmax -logits = model(input_values).logits -predicted_ids = torch.argmax(logits, dim=-1) - -# transcribe -transcription = processor.decode(predicted_ids[0]) - -# FINE-TUNE - -target_transcription = "A MAN SAID TO THE UNIVERSE I EXIST" - -# encode labels -with processor.as_target_processor(): - labels = processor(target_transcription, return_tensors="pt").input_ids - -# compute loss by passing labels -loss = model(input_values, labels=labels).loss -loss.backward() -``` - -# wav2vec - -Example to train a wav2vec model as described in [wav2vec: Unsupervised Pre-training for Speech Recognition (Schneider et al., 2019)](https://arxiv.org/abs/1904.05862). - -## Pre-trained models - -Description | Dataset | Model ----|---|--- -Wav2Vec large | [Librispeech](http://www.openslr.org/12) | [download](https://dl.fbaipublicfiles.com/fairseq/wav2vec/wav2vec_large.pt) - -#### Example usage - -```python -import torch -import fairseq - -cp_path = '/path/to/wav2vec.pt' -model, cfg, task = fairseq.checkpoint_utils.load_model_ensemble_and_task([cp_path]) -model = model[0] -model.eval() - -wav_input_16khz = torch.randn(1,10000) -z = model.feature_extractor(wav_input_16khz) -c = model.feature_aggregator(z) -``` - -## Training a new model with the CLI tools - -Given a directory containing wav files to be used for pretraining (we recommend splitting each file into separate files 10 to 30 seconds in length) - -### Prepare training data manifest - -``` -python examples/wav2vec/wav2vec_manifest.py /path/to/waves --dest /manifest/path --ext wav -``` - -### Train a wav2vec model - -``` -$ python train.py /manifest/path --save-dir /model/path --num-workers 6 --fp16 --max-update 400000 --save-interval 1 --no-epoch-checkpoints \ ---arch wav2vec --task audio_pretraining --min-lr 1e-06 --stop-min-lr 1e-09 --optimizer adam --lr 0.005 --lr-scheduler cosine \ ---conv-feature-layers [(512, 10, 5), (512, 8, 4), (512, 4, 2), (512, 4, 2), (512, 4, 2), (512, 1, 1), (512, 1, 1)] \ ---conv-aggregator-layers [(512, 2, 1), (512, 3, 1), (512, 4, 1), (512, 5, 1), (512, 6, 1), (512, 7, 1), (512, 8, 1), (512, 9, 1), (512, 10, 1), (512, 11, 1), (512, 12, 1), (512, 13, 1)] \ ---skip-connections-agg --residual-scale 0.5 --log-compression --warmup-updates 500 --warmup-init-lr 1e-07 --criterion wav2vec --num-negatives 10 \ ---max-sample-size 150000 --max-tokens 1500000 --skip-invalid-size-inputs-valid-test -``` - -### Run wav2vec2 pre-training on Google Cloud TPUs - -Wav2Vec2 is now supported on TPUs! It's currently pre-training only. - -#### Using hydra on a v3-8 - -``` -$ OMP_NUM_THREADS=1 fairseq-hydra-train \ - task.data=/manifest/path \ - --config-dir /PATH/TO/FAIRSEQ/examples/wav2vec/config/pretraining \ - --config-name wav2vec2_large_librivox_tpu.yaml -``` - -#### Using command line arguments on a v3-8 - -Note: Commandline arguments way of execution has a [known-problem](https://github.com/pytorch/fairseq/issues/3741) currently. - -``` -$ OMP_NUM_THREADS=1 python train.py /manifest/path --save-dir /model/path --num-workers 6 --fp16 --max-update 400000 --save-interval 1 --no-epoch-checkpoints \ ---arch wav2vec2 --task audio_pretraining --min-lr 1e-06 --stop-min-lr 1e-09 --optimizer adam --lr 0.005 --lr-scheduler cosine \ ---conv-feature-layers [(512, 10, 5), (512, 8, 4), (512, 4, 2), (512, 4, 2), (512, 4, 2), (512, 1, 1), (512, 1, 1)] \ ---conv-aggregator-layers [(512, 2, 1), (512, 3, 1), (512, 4, 1), (512, 5, 1), (512, 6, 1), (512, 7, 1), (512, 8, 1), (512, 9, 1), (512, 10, 1), (512, 11, 1), (512, 12, 1), (512, 13, 1)] \ ---skip-connections-agg --residual-scale 0.5 --log-compression --warmup-updates 500 --warmup-init-lr 1e-07 --criterion wav2vec --num-negatives 10 \ ---max-sample-size 150000 --max-tokens 1500000 --skip-invalid-size-inputs-valid-test \ ---tpu --distributed-world-size 8 --num-batch-buckets 3 --enable-padding \ ---encoder-layerdrop 0 --mask-channel-prob 0.1 -``` - -#### Using hydra on a pod slice (v3-N with N > 8) - -``` -$ OMP_NUM_THREADS=1 fairseq-hydra-train \ - task.data=/manifest/path \ - --config-dir /PATH/TO/FAIRSEQ/examples/wav2vec/config/pretraining \ - --config-name wav2vec2_large_librivox_tpu-pod.yaml # edit distributed-world-size accordingly -``` - -#### Using command line arguments on a pod slice (v3-N with N > 8) - -Note: Commandline arguments way of execution has a [known-problem](https://github.com/pytorch/fairseq/issues/3741) currently. - -``` -$ python -m torch_xla.distributed.xla_dist \ - --tpu ${TPUNAME} --conda-env=torch-xla-${TORCH_XLA_VERSION} --env OMP_NUM_THREADS=1 \ - -- \ -python train.py /manifest/path --save-dir /model/path --num-workers 6 --fp16 --max-update 400000 --save-interval 1 --no-epoch-checkpoints \ ---arch wav2vec2 --task audio_pretraining --min-lr 1e-06 --stop-min-lr 1e-09 --optimizer adam --lr 0.005 --lr-scheduler cosine \ ---conv-feature-layers [(512, 10, 5), (512, 8, 4), (512, 4, 2), (512, 4, 2), (512, 4, 2), (512, 1, 1), (512, 1, 1)] \ ---conv-aggregator-layers [(512, 2, 1), (512, 3, 1), (512, 4, 1), (512, 5, 1), (512, 6, 1), (512, 7, 1), (512, 8, 1), (512, 9, 1), (512, 10, 1), (512, 11, 1), (512, 12, 1), (512, 13, 1)] \ ---skip-connections-agg --residual-scale 0.5 --log-compression --warmup-updates 500 --warmup-init-lr 1e-07 --criterion wav2vec --num-negatives 10 \ ---max-sample-size 150000 --max-tokens 1500000 --skip-invalid-size-inputs-valid-test \ ---tpu --distributed-world-size ${WORLD_SIZE} --num-batch-buckets 3 --enable-padding \ ---encoder-layerdrop 0 --mask-channel-prob 0.1 -``` - -### Extract embeddings from the downstream task data - -``` -$ PYTHONPATH=/path/to/fairseq python examples/wav2vec/wav2vec_featurize.py --input /path/to/task/waves --output /path/to/output \ ---model /model/path/checkpoint_best.pt --split train valid test -``` - -# vq-wav2vec - -Example to train a vq-wav2vec model as described in [vq-wav2vec: Self-Supervised Learning of Discrete Speech Representations (Baevski et al., 2019)](https://arxiv.org/abs/1910.05453). - -These models are also used in [Effectiveness of self-supervised pre-training for speech recognition (Baevski et al., 2019)](https://arxiv.org/abs/1911.03912). - -## Pre-trained models - -Description | Dataset | Model ----|---|--- -vq-wav2vec Gumbel | [Librispeech](http://www.openslr.org/12) | [download](https://dl.fbaipublicfiles.com/fairseq/wav2vec/vq-wav2vec.pt) -vq-wav2vec K-means | [Librispeech](http://www.openslr.org/12) | [download](https://dl.fbaipublicfiles.com/fairseq/wav2vec/vq-wav2vec_kmeans.pt) -Roberta on K-means codes | [Librispeech](http://www.openslr.org/12) | [download](https://dl.fbaipublicfiles.com/fairseq/wav2vec/bert_kmeans.tar) - -#### Example usage - -```python -import torch -import fairseq - -cp = torch.load('/path/to/vq-wav2vec.pt') -model, cfg, task = fairseq.checkpoint_utils.load_model_ensemble_and_task([cp]) -model = model[0] -model.eval() - -wav_input_16khz = torch.randn(1,10000) -z = model.feature_extractor(wav_input_16khz) -_, idxs = model.vector_quantizer.forward_idx(z) -print(idxs.shape) # output: torch.Size([1, 60, 2]), 60 timesteps with 2 indexes corresponding to 2 groups in the model -``` - -## Training a new model with the CLI tools - -Given a directory containing wav files to be used for pretraining (we recommend splitting each file into separate file 10 to 30 seconds in length) - -### Prepare training data manifest - -``` -python examples/wav2vec/wav2vec_manifest.py /path/to/waves --dest /manifest/path --ext wav -``` - -### Train a gumbel vq-wav2vec model - -``` -$ python train.py /manifest/path --save-dir /model/path --num-workers 6 --fp16 --max-update 400000 \ ---save-interval 1 --no-epoch-checkpoints --arch wav2vec --task audio_pretraining --min-lr 1e-06 --stop-min-lr 1e-09 \ ---optimizer adam --lr 1e-05 --lr-scheduler cosine \ ---conv-feature-layers [(512, 10, 5), (512, 8, 4), (512, 4, 2), (512, 4, 2), (512, 4, 2), (512, 1, 1), (512, 1, 1), (512, 1, 1)] \ ---conv-aggregator-layers [(512, 2, 1), (512, 3, 1), (512, 4, 1), (512, 5, 1), (512, 6, 1), (512, 7, 1), (512, 8, 1), (512, 9, 1), (512, 10, 1), (512, 11, 1), (512, 12, 1), (512, 13, 1)] \ ---activation gelu --offset auto --skip-connections-agg --residual-scale 0.5 \ ---log-keys ["prob_perplexity","code_perplexity","temp"] --vq-type gumbel --vq-groups 2 --vq-depth 2 \ ---combine-groups --vq-vars 320 --vq-temp (2,0.5,0.999995) --prediction-steps 12 --warmup-updates 1000 \ ---warmup-init-lr 1e-07 --criterion wav2vec --num-negatives 10 --max-sample-size 150000 \ ---max-tokens 300000 --cross-sample-negatives 0 --update-freq 1 --seed 2 --skip-invalid-size-inputs-valid-test -``` - -for k-means training, set vq-type with "kmeans" and add --loss-weights [1] argument. Pre-trained models were trained on 16 GPUs. - -### Tokenize audio data (e.g. for BERT training) - -``` -$ PYTHONPATH=/path/to/fairseq python examples/wav2vec/vq-wav2vec_featurize.py --data-dir /manifest/path --output-dir /path/to/output \ ---checkpoint /model/path/checkpoint_best.pt --split train valid test --extension tsv -``` diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/finetuning/base_100h.yaml b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/finetuning/base_100h.yaml deleted file mode 100644 index 153b5df1708f187933a68ff55009652a107c69eb..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/finetuning/base_100h.yaml +++ /dev/null @@ -1,58 +0,0 @@ -# @package _group_ - -common: - fp16: true - log_format: json - log_interval: 200 - -checkpoint: - no_epoch_checkpoints: true - best_checkpoint_metric: wer - -task: - _name: audio_finetuning - data: ??? - normalize: false - labels: ltr - -dataset: - num_workers: 6 - max_tokens: 3200000 - skip_invalid_size_inputs_valid_test: true - valid_subset: dev_other - -distributed_training: - ddp_backend: legacy_ddp - distributed_world_size: 2 - -criterion: - _name: ctc - zero_infinity: true - -optimization: - max_update: 80000 - lr: [0.00003] - sentence_avg: true - update_freq: [4] - -optimizer: - _name: adam - adam_betas: (0.9,0.98) - adam_eps: 1e-08 - -lr_scheduler: - _name: tri_stage - phase_ratio: [0.1, 0.4, 0.5] - final_lr_scale: 0.05 - -model: - _name: wav2vec_ctc - w2v_path: ??? - apply_mask: true - mask_prob: 0.65 - mask_channel_prob: 0.5 - mask_channel_length: 64 - layerdrop: 0.1 - activation_dropout: 0.1 - feature_grad_mult: 0.0 - freeze_finetune_updates: 0 diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/finetuning/base_10h.yaml b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/finetuning/base_10h.yaml deleted file mode 100644 index 5044518025aadf73b3f5f204a74969cf3f982f03..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/finetuning/base_10h.yaml +++ /dev/null @@ -1,63 +0,0 @@ -# @package _group_ - -common: - fp16: true - log_format: json - log_interval: 200 - -checkpoint: - save_interval: 50 - save_interval_updates: 10000 - keep_interval_updates: 1 - no_epoch_checkpoints: true - best_checkpoint_metric: wer - -task: - _name: audio_finetuning - data: ??? - normalize: false - labels: ltr - -dataset: - num_workers: 6 - max_tokens: 3200000 - skip_invalid_size_inputs_valid_test: true - validate_after_updates: 10000 - validate_interval: 50 - valid_subset: dev_other - -distributed_training: - ddp_backend: legacy_ddp - distributed_world_size: 2 - -criterion: - _name: ctc - zero_infinity: true - -optimization: - max_update: 20000 - lr: [0.00005] - sentence_avg: true - update_freq: [4] - -optimizer: - _name: adam - adam_betas: (0.9,0.98) - adam_eps: 1e-08 - -lr_scheduler: - _name: tri_stage - phase_ratio: [0.1, 0.4, 0.5] - final_lr_scale: 0.05 - -model: - _name: wav2vec_ctc - w2v_path: ??? - apply_mask: true - mask_prob: 0.65 - mask_channel_prob: 0.5 - mask_channel_length: 64 - layerdrop: 0.05 - activation_dropout: 0.1 - feature_grad_mult: 0.0 - freeze_finetune_updates: 10000 diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/finetuning/base_10m.yaml b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/finetuning/base_10m.yaml deleted file mode 100644 index 14abc013bd1862bc0ec3df7e2603924ab9fe671a..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/finetuning/base_10m.yaml +++ /dev/null @@ -1,63 +0,0 @@ -# @package _group_ - -common: - fp16: true - log_format: json - log_interval: 200 - -checkpoint: - save_interval: 1000 - save_interval_updates: 50 - keep_interval_updates: 1 - no_epoch_checkpoints: true - best_checkpoint_metric: wer - -task: - _name: audio_finetuning - data: ??? - normalize: false - labels: ltr - -dataset: - num_workers: 6 - max_tokens: 3200000 - skip_invalid_size_inputs_valid_test: true - validate_after_updates: 10000 - validate_interval: 1000 - valid_subset: dev_other - -distributed_training: - ddp_backend: legacy_ddp - distributed_world_size: 2 - -criterion: - _name: ctc - zero_infinity: true - -optimization: - max_update: 13000 - lr: [0.00005] - sentence_avg: true - update_freq: [4] - -optimizer: - _name: adam - adam_betas: (0.9,0.98) - adam_eps: 1e-08 - -lr_scheduler: - _name: tri_stage - phase_ratio: [0.1, 0.4, 0.5] - final_lr_scale: 0.05 - -model: - _name: wav2vec_ctc - w2v_path: ??? - apply_mask: true - mask_prob: 0.65 - mask_channel_prob: 0.25 - mask_channel_length: 64 - layerdrop: 0.1 - activation_dropout: 0.1 - feature_grad_mult: 0.0 - freeze_finetune_updates: 10000 diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/finetuning/base_1h.yaml b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/finetuning/base_1h.yaml deleted file mode 100644 index a0af1cfad7e9f5ac4d1f1332852fb2022ea9ca74..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/finetuning/base_1h.yaml +++ /dev/null @@ -1,63 +0,0 @@ -# @package _group_ - -common: - fp16: true - log_format: json - log_interval: 200 - -checkpoint: - save_interval: 50 - save_interval_updates: 1000 - keep_interval_updates: 1 - no_epoch_checkpoints: true - best_checkpoint_metric: wer - -task: - _name: audio_finetuning - data: ??? - normalize: false - labels: ltr - -dataset: - num_workers: 6 - max_tokens: 3200000 - skip_invalid_size_inputs_valid_test: true - validate_after_updates: 10000 - validate_interval: 1000 - valid_subset: dev_other - -distributed_training: - ddp_backend: legacy_ddp - distributed_world_size: 2 - -criterion: - _name: ctc - zero_infinity: true - -optimization: - max_update: 13000 - lr: [0.00005] - sentence_avg: true - update_freq: [4] - -optimizer: - _name: adam - adam_betas: (0.9,0.98) - adam_eps: 1e-08 - -lr_scheduler: - _name: tri_stage - phase_ratio: [0.1, 0.4, 0.5] - final_lr_scale: 0.05 - -model: - _name: wav2vec_ctc - w2v_path: ??? - apply_mask: true - mask_prob: 0.65 - mask_channel_prob: 0.25 - mask_channel_length: 64 - layerdrop: 0.1 - activation_dropout: 0.1 - feature_grad_mult: 0.0 - freeze_finetune_updates: 10000 diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/finetuning/base_960h.yaml b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/finetuning/base_960h.yaml deleted file mode 100644 index 3eadc36b37b5caa7c0f0838e4e9ff2d401110429..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/finetuning/base_960h.yaml +++ /dev/null @@ -1,57 +0,0 @@ -# @package _group_ - -common: - fp16: true - log_format: json - log_interval: 200 - -checkpoint: - no_epoch_checkpoints: true - best_checkpoint_metric: wer - -task: - _name: audio_finetuning - data: ??? - normalize: false - labels: ltr - -dataset: - num_workers: 6 - max_tokens: 3200000 - skip_invalid_size_inputs_valid_test: true - valid_subset: dev_other - -distributed_training: - ddp_backend: legacy_ddp - distributed_world_size: 8 - -criterion: - _name: ctc - zero_infinity: true - -optimization: - max_update: 320000 - lr: [0.0001] - sentence_avg: true - -optimizer: - _name: adam - adam_betas: (0.9,0.98) - adam_eps: 1e-08 - -lr_scheduler: - _name: tri_stage - phase_ratio: [0.1, 0.4, 0.5] - final_lr_scale: 0.05 - -model: - _name: wav2vec_ctc - w2v_path: ??? - apply_mask: true - mask_prob: 0.5 - mask_channel_prob: 0.1 - mask_channel_length: 64 - layerdrop: 0.1 - activation_dropout: 0.1 - feature_grad_mult: 0.0 - freeze_finetune_updates: 0 diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/finetuning/vox_100h.yaml b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/finetuning/vox_100h.yaml deleted file mode 100644 index b8f81e5e1895e448c6126f9ccd6d9c4a5fb36b18..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/finetuning/vox_100h.yaml +++ /dev/null @@ -1,58 +0,0 @@ -# @package _group_ - -common: - fp16: true - log_format: json - log_interval: 200 - -checkpoint: - no_epoch_checkpoints: true - best_checkpoint_metric: wer - -task: - _name: audio_finetuning - data: ??? - normalize: true - labels: ltr - -dataset: - num_workers: 6 - max_tokens: 1280000 - skip_invalid_size_inputs_valid_test: true - valid_subset: dev_other - -distributed_training: - ddp_backend: legacy_ddp - distributed_world_size: 4 - -criterion: - _name: ctc - zero_infinity: true - -optimization: - max_update: 80000 - lr: [0.00003] - sentence_avg: true - update_freq: [5] - -optimizer: - _name: adam - adam_betas: (0.9,0.98) - adam_eps: 1e-08 - -lr_scheduler: - _name: tri_stage - phase_ratio: [0.1, 0.4, 0.5] - final_lr_scale: 0.05 - -model: - _name: wav2vec_ctc - w2v_path: ??? - apply_mask: true - mask_prob: 0.5 - mask_channel_prob: 0.5 - mask_channel_length: 64 - layerdrop: 0.1 - activation_dropout: 0.1 - feature_grad_mult: 0.0 - freeze_finetune_updates: 10000 diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/finetuning/vox_10h.yaml b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/finetuning/vox_10h.yaml deleted file mode 100644 index 8f1ca71ee2bd195934f85010e141772479769f0b..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/finetuning/vox_10h.yaml +++ /dev/null @@ -1,63 +0,0 @@ -# @package _group_ - -common: - fp16: true - log_format: json - log_interval: 200 - -checkpoint: - save_interval: 50 - save_interval_updates: 10000 - keep_interval_updates: 1 - no_epoch_checkpoints: true - best_checkpoint_metric: wer - -task: - _name: audio_finetuning - data: ??? - normalize: true - labels: ltr - -dataset: - num_workers: 6 - max_tokens: 1280000 - skip_invalid_size_inputs_valid_test: true - validate_after_updates: 10000 - validate_interval: 50 - valid_subset: dev_other - -distributed_training: - ddp_backend: legacy_ddp - distributed_world_size: 4 - -criterion: - _name: ctc - zero_infinity: true - -optimization: - max_update: 20000 - lr: [0.0001] - sentence_avg: true - update_freq: [5] - -optimizer: - _name: adam - adam_betas: (0.9,0.98) - adam_eps: 1e-08 - -lr_scheduler: - _name: tri_stage - phase_ratio: [0.1, 0.4, 0.5] - final_lr_scale: 0.05 - -model: - _name: wav2vec_ctc - w2v_path: ??? - apply_mask: true - mask_prob: 0.75 - mask_channel_prob: 0.25 - mask_channel_length: 64 - layerdrop: 0.1 - activation_dropout: 0.1 - feature_grad_mult: 0.0 - freeze_finetune_updates: 10000 diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/finetuning/vox_10m.yaml b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/finetuning/vox_10m.yaml deleted file mode 100644 index 07e327fe7477c88545d7744a5928d2fe703dd6de..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/finetuning/vox_10m.yaml +++ /dev/null @@ -1,63 +0,0 @@ -# @package _group_ - -common: - fp16: true - log_format: json - log_interval: 200 - -checkpoint: - save_interval: 1000 - save_interval_updates: 50 - keep_interval_updates: 1 - no_epoch_checkpoints: true - best_checkpoint_metric: wer - -task: - _name: audio_finetuning - data: ??? - normalize: true - labels: ltr - -dataset: - num_workers: 6 - max_tokens: 1280000 - skip_invalid_size_inputs_valid_test: true - validate_after_updates: 10000 - validate_interval: 1000 - valid_subset: dev_other - -distributed_training: - ddp_backend: legacy_ddp - distributed_world_size: 4 - -criterion: - _name: ctc - zero_infinity: true - -optimization: - max_update: 13000 - lr: [0.0001] - sentence_avg: true - update_freq: [5] - -optimizer: - _name: adam - adam_betas: (0.9,0.98) - adam_eps: 1e-08 - -lr_scheduler: - _name: tri_stage - phase_ratio: [0.1, 0.4, 0.5] - final_lr_scale: 0.05 - -model: - _name: wav2vec_ctc - w2v_path: ??? - apply_mask: true - mask_prob: 0.65 - mask_channel_prob: 0.25 - mask_channel_length: 64 - layerdrop: 0.1 - activation_dropout: 0.1 - feature_grad_mult: 0.0 - freeze_finetune_updates: 10000 diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/finetuning/vox_1h.yaml b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/finetuning/vox_1h.yaml deleted file mode 100644 index fac1bbb32fa98364568dd1af0f6fb703906e0606..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/finetuning/vox_1h.yaml +++ /dev/null @@ -1,63 +0,0 @@ -# @package _group_ - -common: - fp16: true - log_format: json - log_interval: 200 - -checkpoint: - save_interval: 1000 - save_interval_updates: 50 - keep_interval_updates: 1 - no_epoch_checkpoints: true - best_checkpoint_metric: wer - -task: - _name: audio_finetuning - data: ??? - normalize: true - labels: ltr - -dataset: - num_workers: 6 - max_tokens: 1280000 - skip_invalid_size_inputs_valid_test: true - validate_after_updates: 10000 - validate_interval: 1000 - valid_subset: dev_other - -distributed_training: - ddp_backend: legacy_ddp - distributed_world_size: 4 - -criterion: - _name: ctc - zero_infinity: true - -optimization: - max_update: 13000 - lr: [0.0003] - sentence_avg: true - update_freq: [5] - -optimizer: - _name: adam - adam_betas: (0.9,0.98) - adam_eps: 1e-08 - -lr_scheduler: - _name: tri_stage - phase_ratio: [0.1, 0.4, 0.5] - final_lr_scale: 0.05 - -model: - _name: wav2vec_ctc - w2v_path: ??? - apply_mask: true - mask_prob: 0.75 - mask_channel_prob: 0.25 - mask_channel_length: 64 - layerdrop: 0.1 - activation_dropout: 0.1 - feature_grad_mult: 0.0 - freeze_finetune_updates: 10000 diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/finetuning/vox_960h.yaml b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/finetuning/vox_960h.yaml deleted file mode 100644 index 9d72404fa3b72624ed00556deced2513dff075d8..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/finetuning/vox_960h.yaml +++ /dev/null @@ -1,57 +0,0 @@ -# @package _group_ - -common: - fp16: true - log_format: json - log_interval: 200 - -checkpoint: - no_epoch_checkpoints: true - best_checkpoint_metric: wer - -task: - _name: audio_finetuning - data: ??? - normalize: true - labels: ltr - -dataset: - num_workers: 6 - max_tokens: 1280000 - skip_invalid_size_inputs_valid_test: true - valid_subset: dev_other - -distributed_training: - ddp_backend: legacy_ddp - distributed_world_size: 24 - -criterion: - _name: ctc - zero_infinity: true - -optimization: - max_update: 320000 - lr: [0.00003] - sentence_avg: true - -optimizer: - _name: adam - adam_betas: (0.9,0.98) - adam_eps: 1e-08 - -lr_scheduler: - _name: tri_stage - phase_ratio: [0.1, 0.4, 0.5] - final_lr_scale: 0.05 - -model: - _name: wav2vec_ctc - w2v_path: ??? - apply_mask: true - mask_prob: 0.5 - mask_channel_prob: 0.25 - mask_channel_length: 64 - layerdrop: 0.1 - activation_dropout: 0.1 - feature_grad_mult: 0.0 - freeze_finetune_updates: 10000 diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/pretraining/wav2vec2_base_librispeech.yaml b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/pretraining/wav2vec2_base_librispeech.yaml deleted file mode 100644 index b686e21ab1d367158fe7afa4197303a4ee74df66..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/pretraining/wav2vec2_base_librispeech.yaml +++ /dev/null @@ -1,57 +0,0 @@ -# @package _group_ - -common: - fp16: true - log_format: json - log_interval: 200 - -checkpoint: - save_interval_updates: 25000 - keep_interval_updates: 1 - no_epoch_checkpoints: true - -task: - _name: audio_pretraining - data: ??? - max_sample_size: 250000 - min_sample_size: 32000 - normalize: false - -dataset: - num_workers: 6 - max_tokens: 1400000 - skip_invalid_size_inputs_valid_test: true - -distributed_training: - distributed_world_size: 64 - ddp_backend: legacy_ddp - -criterion: - _name: wav2vec - infonce: true - log_keys: ["prob_perplexity","code_perplexity","temp"] - loss_weights: [0.1, 10] - -optimization: - max_update: 400000 - lr: [0.0005] - -optimizer: - _name: adam - adam_betas: (0.9,0.98) - adam_eps: 1e-06 - weight_decay: 0.01 - -lr_scheduler: - _name: polynomial_decay - warmup_updates: 32000 - -model: - _name: wav2vec2 - quantize_targets: true - final_dim: 256 - encoder_layerdrop: 0.05 - dropout_input: 0.1 - dropout_features: 0.1 - feature_grad_mult: 0.1 - encoder_embed_dim: 768 diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/pretraining/wav2vec2_conformer_base_librispeech.yaml b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/pretraining/wav2vec2_conformer_base_librispeech.yaml deleted file mode 100644 index 912ac152fd27c6ac51ae648950ee53d8a7226a75..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/pretraining/wav2vec2_conformer_base_librispeech.yaml +++ /dev/null @@ -1,60 +0,0 @@ -# @package _group_ - -common: - fp16: true - log_format: json - log_interval: 200 - -checkpoint: - save_interval_updates: 25000 - keep_interval_updates: 1 - no_epoch_checkpoints: true - -task: - _name: audio_pretraining - data: ??? - max_sample_size: 250000 - min_sample_size: 32000 - normalize: false - -dataset: - num_workers: 6 - max_tokens: 1400000 - skip_invalid_size_inputs_valid_test: true - -distributed_training: - distributed_world_size: 64 - ddp_backend: legacy_ddp - -criterion: - _name: wav2vec - infonce: true - log_keys: ["prob_perplexity","code_perplexity","temp"] - loss_weights: [0.1, 10] - -optimization: - max_update: 400000 - lr: [0.0005] - -optimizer: - _name: adam - adam_betas: (0.9,0.98) - adam_eps: 1e-06 - weight_decay: 0.01 - -lr_scheduler: - _name: polynomial_decay - warmup_updates: 32000 - -model: - _name: wav2vec2 - quantize_targets: true - final_dim: 256 - encoder_layerdrop: 0.05 - dropout_input: 0.1 - dropout_features: 0.1 - feature_grad_mult: 0.1 - encoder_embed_dim: 768 - layer_type: conformer - attn_type: espnet - pos_enc_type: rel_pos diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/pretraining/wav2vec2_conformer_large_librivox.yaml b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/pretraining/wav2vec2_conformer_large_librivox.yaml deleted file mode 100644 index 676166b6b767cf52cbb819a732f943445763140b..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/pretraining/wav2vec2_conformer_large_librivox.yaml +++ /dev/null @@ -1,72 +0,0 @@ -# @package _group_ - -common: - fp16: true - log_format: json - log_interval: 200 - -checkpoint: - save_interval_updates: 25000 - keep_interval_updates: 1 - no_epoch_checkpoints: true - -task: - _name: audio_pretraining - data: ??? - max_sample_size: 320000 - min_sample_size: 32000 - normalize: true - -dataset: - num_workers: 6 - max_tokens: 1200000 - skip_invalid_size_inputs_valid_test: true - -distributed_training: - distributed_world_size: 128 - ddp_backend: legacy_ddp - -criterion: - _name: wav2vec - infonce: true - log_keys: ["prob_perplexity","code_perplexity","temp"] - loss_weights: [0.1, 0] - -optimization: - max_update: 1000000 - lr: [0.005] - -optimizer: - _name: adam - adam_betas: (0.9,0.98) - adam_eps: 1e-06 - weight_decay: 0.01 - -lr_scheduler: - _name: polynomial_decay - warmup_updates: 32000 - -model: - _name: wav2vec2 - quantize_targets: true - extractor_mode: layer_norm - layer_norm_first: true - final_dim: 768 - latent_temp: [2.0,0.1,0.999995] - encoder_layerdrop: 0.00 - dropout_input: 0.0 - dropout_features: 0.0 - dropout: 0.0 - attention_dropout: 0.0 - conv_bias: true - - encoder_layers: 24 - encoder_embed_dim: 1024 - encoder_ffn_embed_dim: 4096 - encoder_attention_heads: 16 - - feature_grad_mult: 1.0 - - layer_type: conformer - attn_type: espnet - pos_enc_type: rel_pos diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/pretraining/wav2vec2_large_librivox.yaml b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/pretraining/wav2vec2_large_librivox.yaml deleted file mode 100644 index 3192ce4cba42d1741099d27aad8588a45a59814c..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/pretraining/wav2vec2_large_librivox.yaml +++ /dev/null @@ -1,70 +0,0 @@ -# @package _group_ - -common: - fp16: true - log_format: json - log_interval: 200 - -checkpoint: - save_interval_updates: 25000 - keep_interval_updates: 1 - no_epoch_checkpoints: true - -task: - _name: audio_pretraining - data: ??? - max_sample_size: 320000 - min_sample_size: 32000 - normalize: true - -dataset: - batch_size: 4 - num_workers: 6 - max_tokens: 1200000 - skip_invalid_size_inputs_valid_test: true - -distributed_training: - distributed_world_size: 128 - ddp_backend: legacy_ddp - -criterion: - _name: wav2vec - infonce: true - log_keys: ["prob_perplexity","code_perplexity","temp"] - loss_weights: [0.1, 0] - -optimization: - max_update: 1000000 - lr: [0.005] - -optimizer: - _name: adam - adam_betas: (0.9,0.98) - adam_eps: 1e-06 - weight_decay: 0.01 - -lr_scheduler: - _name: polynomial_decay - warmup_updates: 32000 - -model: - _name: wav2vec2 - quantize_targets: true - extractor_mode: layer_norm - layer_norm_first: true - final_dim: 768 - latent_temp: [2.0,0.1,0.999995] - encoder_layerdrop: 0.00 - dropout_input: 0.0 - dropout_features: 0.0 - dropout: 0.0 - attention_dropout: 0.0 - conv_bias: true - - encoder_layers: 24 - encoder_embed_dim: 1024 - encoder_ffn_embed_dim: 4096 - encoder_attention_heads: 16 - - feature_grad_mult: 1.0 - diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/pretraining/wav2vec2_large_librivox_tpu-pod.yaml b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/pretraining/wav2vec2_large_librivox_tpu-pod.yaml deleted file mode 100644 index ff35a95b6596b74215ef1bbdd2ec8d462d1d8542..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/pretraining/wav2vec2_large_librivox_tpu-pod.yaml +++ /dev/null @@ -1,72 +0,0 @@ -# @package _group_ - -common: - tpu: true - fp16: false - log_format: json - log_interval: 10 - -checkpoint: - save_interval_updates: 25000 - keep_interval_updates: 1 - no_epoch_checkpoints: true - -task: - _name: audio_pretraining - data: ??? - max_sample_size: 250000 - min_sample_size: 32000 - normalize: true - num_batch_buckets: 3 - precompute_mask_indices: true - enable_padding: true - -dataset: - num_workers: 6 - max_tokens: 1200000 - skip_invalid_size_inputs_valid_test: true - -distributed_training: - distributed_world_size: 128 - ddp_backend: legacy_ddp - -criterion: - _name: wav2vec - infonce: true - log_keys: ["prob_perplexity","code_perplexity","temp"] - loss_weights: [0.1, 0] - -optimization: - max_update: 1000000 - lr: [0.005] - -optimizer: - _name: adam - adam_betas: (0.9,0.98) - adam_eps: 1e-06 - weight_decay: 0.01 - -lr_scheduler: - _name: polynomial_decay - warmup_updates: 32000 - -model: - _name: wav2vec2 - quantize_targets: true - extractor_mode: layer_norm - layer_norm_first: true - final_dim: 768 - latent_temp: [2.0,0.1,0.999995] - encoder_layerdrop: 0.00 - dropout_input: 0.0 - dropout_features: 0.0 - dropout: 0.0 - attention_dropout: 0.0 - conv_bias: true - - encoder_layers: 24 - encoder_embed_dim: 1024 - encoder_ffn_embed_dim: 4096 - encoder_attention_heads: 16 - - feature_grad_mult: 1.0 diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/pretraining/wav2vec2_large_librivox_tpu.yaml b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/pretraining/wav2vec2_large_librivox_tpu.yaml deleted file mode 100644 index ee55bdab72e40c2f410cde592e3cfe5ff7f06e0c..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/config/pretraining/wav2vec2_large_librivox_tpu.yaml +++ /dev/null @@ -1,77 +0,0 @@ -# @package _group_ - -common: - tpu: true - fp16: false - log_format: json - log_interval: 10 - -checkpoint: - save_interval_updates: 25000 - keep_interval_updates: 1 - no_epoch_checkpoints: true - -task: - _name: audio_pretraining - data: ??? - max_sample_size: 250000 - min_sample_size: 32000 - normalize: true - num_batch_buckets: 3 - precompute_mask_indices: true - enable_padding: true - inferred_w2v_config: - mask_prob: 0.65 - mask_selection: 'static' - mask_other: 0 - mask_channel_prob: 0.1 - -dataset: - num_workers: 6 - max_tokens: 1200000 - skip_invalid_size_inputs_valid_test: true - -distributed_training: - distributed_world_size: 8 - ddp_backend: legacy_ddp - -criterion: - _name: wav2vec - infonce: true - log_keys: ["prob_perplexity","code_perplexity","temp"] - loss_weights: [0.1, 0] - -optimization: - max_update: 1000000 - lr: [0.005] - -optimizer: - _name: adam - adam_betas: (0.9,0.98) - adam_eps: 1e-06 - weight_decay: 0.01 - -lr_scheduler: - _name: polynomial_decay - warmup_updates: 32000 - -model: - _name: wav2vec2 - quantize_targets: true - extractor_mode: layer_norm - layer_norm_first: true - final_dim: 768 - latent_temp: [2.0,0.1,0.999995] - encoder_layerdrop: 0.00 - dropout_input: 0.0 - dropout_features: 0.0 - dropout: 0.0 - attention_dropout: 0.0 - conv_bias: true - - encoder_layers: 24 - encoder_embed_dim: 1024 - encoder_ffn_embed_dim: 4096 - encoder_attention_heads: 16 - - feature_grad_mult: 1.0 diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/libri_labels.py b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/libri_labels.py deleted file mode 100644 index 694a202604c7a4a480550550679ce6c16bd10e42..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/libri_labels.py +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -""" -Helper script to pre-compute embeddings for a flashlight (previously called wav2letter++) dataset -""" - -import argparse -import os - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("tsv") - parser.add_argument("--output-dir", required=True) - parser.add_argument("--output-name", required=True) - args = parser.parse_args() - - os.makedirs(args.output_dir, exist_ok=True) - - transcriptions = {} - - with open(args.tsv, "r") as tsv, open( - os.path.join(args.output_dir, args.output_name + ".ltr"), "w" - ) as ltr_out, open( - os.path.join(args.output_dir, args.output_name + ".wrd"), "w" - ) as wrd_out: - root = next(tsv).strip() - for line in tsv: - line = line.strip() - dir = os.path.dirname(line) - if dir not in transcriptions: - parts = dir.split(os.path.sep) - trans_path = f"{parts[-2]}-{parts[-1]}.trans.txt" - path = os.path.join(root, dir, trans_path) - assert os.path.exists(path) - texts = {} - with open(path, "r") as trans_f: - for tline in trans_f: - items = tline.strip().split() - texts[items[0]] = " ".join(items[1:]) - transcriptions[dir] = texts - part = os.path.basename(line).split(".")[0] - assert part in transcriptions[dir] - print(transcriptions[dir][part], file=wrd_out) - print( - " ".join(list(transcriptions[dir][part].replace(" ", "|"))) + " |", - file=ltr_out, - ) - - -if __name__ == "__main__": - main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/scripts/binarize_manifest.sh b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/scripts/binarize_manifest.sh deleted file mode 100644 index 6f201bdb524fad51a69d8c45889eaa1578efc62d..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/scripts/binarize_manifest.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -# usage: bash binarize_manifest <dest_dir> <train_split> <valid_split> - -DEST_DIR=$1 -TRAIN_SPLIT=$2 -VALID_SPLIT=$3 -FAIRSEQ_ROOT=$4 - -mkdir -p $DEST_DIR - -# split file path and lengths into separate files -cut -f1 $TRAIN_SPLIT.tsv > $DEST_DIR/train_fnames.txt -cut -f1 $VALID_SPLIT.tsv > $DEST_DIR/valid_fnames.txt -cut -f2 $TRAIN_SPLIT.tsv > $DEST_DIR/train.lengths -cut -f2 $VALID_SPLIT.tsv > $DEST_DIR/valid.lengths - -# copy root directory -head -1 $TRAIN_SPLIT.tsv > $DEST_DIR/train.root -head -1 $VALID_SPLIT.tsv > $DEST_DIR/valid.root - -# remove root directory -sed -i '1d' $DEST_DIR/train_fnames.txt -sed -i '1d' $DEST_DIR/valid_fnames.txt -sed -i '1d' $DEST_DIR/train.lengths -sed -i '1d' $DEST_DIR/valid.lengths - -# insert spaces between characters -sed -i -e 's/\(.\)/\1 /g' $DEST_DIR/train_fnames.txt -sed -i -e 's/\(.\)/\1 /g' $DEST_DIR/valid_fnames.txt - -# run preprocessor -PYTHONPATH=$FAIRSEQ_ROOT python $FAIRSEQ_ROOT/fairseq_cli/preprocess.py --dataset-impl mmap --trainpref $DEST_DIR/train_fnames.txt --validpref $DEST_DIR/valid_fnames.txt --workers 60 --only-source --destdir $DEST_DIR diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/README.md b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/README.md deleted file mode 100644 index 0b213fd202d04bce2149936ec149c23c6d483745..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/README.md +++ /dev/null @@ -1,103 +0,0 @@ -# wav2vec Unsupervised (wav2vec-U) - -Wav2vec Unsupervised (wav2vec-U) is a framework for building speech recognition systems without any labeled training data as described in [Unsupervised Speech Recognition (Baevski et al., 2021)](https://ai.facebook.com/research/publications/unsupervised-speech-recognition). The model takes as input wav2vec 2.0 or XLSR representations (see [pretrained models](https://github.com/pytorch/fairseq/blob/main/examples/wav2vec)) as well as unlabeled speech and text data. - - The wav2vec-U training procedure consists of three consecutive main steps: -* Preparation of speech representations and text data -* Generative adversarial training (GAN) -* Iterative self-training + Kaldi LM-decoding - -## Preparation of speech and text data -Similar to [wav2vec 2.0](https://github.com/pytorch/fairseq/blob/main/examples/wav2vec/README.md), data folders contain {train,valid,test}.{tsv,wrd,phn} files, where audio paths are stored in tsv files, and word, letter or phoneme transcriptions are stored in .{wrd,ltr,phn}. - -In **/path/to/data/with_silence** you need a *train.tsv* file as well as (optionally) *{valid,test}.{tsv,wrd,phn}*. It is nice to have *10h.{tsv,phn}* files there too for reproducing the ablation study on layer selection. In **/path/to/data/without_silence** you have the same files, except *.tsv* files contain audios with silences removed using rVAD. - -Pre-requisites: -* set FAIRSEQ_ROOT environmental variable to your fairseq installation -* set RVAD_ROOT environmental variable to a checkout of [rVADfast](https://github.com/zhenghuatan/rVADfast) -* set KENLM_ROOT environmental variable to the location of [KenLM](https://github.com/kpu/kenlm) binaries -* install [PyKaldi](https://github.com/pykaldi/pykaldi) and set KALDI_ROOT environmental variable to the location of your kaldi installation. To use the version bundled with PyKaldi, you can use /path/to/pykaldi/tools/kaldi - -Create new audio files without silences: -```shell -# create a manifest file for the set original of audio files -python $FAIRSEQ_ROOT/examples/wav2vec/wav2vec_manifest.py /dir/to/save/audio/files --ext wav --dest /path/to/new/train.tsv --valid-percent 0 - -python scripts/vads.py -r $RVAD_ROOT < /path/to/train.tsv > train.vads - -python scripts/remove_silence.py --tsv /path/to/train.tsv --vads train.vads --out /dir/to/save/audio/files - -python $FAIRSEQ_ROOT/examples/wav2vec/wav2vec_manifest.py /dir/to/save/audio/files --ext wav --dest /path/to/new/train.tsv --valid-percent 0.01 -``` - -Next, we need to preprocess the audio data to better match phonemized text data: - -```shell -zsh scripts/prepare_audio.sh /dir/with/{train,test,valid}.tsv /output/dir /path/to/wav2vec2/model.pt 512 14 -``` -Note that if you have splits different than train/valid/test, you will need to modify this script. The last two arguments are the PCA dimensionality and the 0-based index of the layer from which to extract representations. - -Now we need to prepare text data: -```shell -zsh scripts/prepare_text.sh language /path/to/text/file /output/dir 1000 espeak /path/to/fasttext/lid/model -``` - -The fourth argument is minimum number observations of phones to keep. If your text corpus is small, you might want to reduce this number. - -The fifth argument is which phonemizer to use. Supported values are [espeak](http://espeak.sourceforge.net/), [espeak-ng](https://github.com/espeak-ng/espeak-ng), and [G2P](https://github.com/Kyubyong/g2p) (english only). - -Pre-trained fasttext LID models can be downloaded [here](https://fasttext.cc/docs/en/language-identification.html). - -### Prepare TIMIT data -TIMIT transcripts include silence. Therefore VAD is not used for audio preprocessing, and we do not wrap transcripts with silences or insert random silence in between words. - -To prepare TIMIT data for both the matched an unmatched setup: -```shell -bash scripts/prepare_timit.sh /dir/to/timit/raw/data /output/dir /path/to/wav2vec2/model.pt -``` - -Note that we assume the TIMIT distribution with capitalized directories and filenames are used (e.g., `TRAIN/DR1/FCJF0/SA1.PHN`). - -## Generative adversarial training (GAN) - -We then use a GAN model to build a first unsupervised ASR model. The data preparation above of both speech features and text data is a necessary procedure that enables the generator to match speech to text in an unsupervised way. - -Launching GAN training on top of preprocessed features, with default hyperparameters can be done with: - -``` -PREFIX=w2v_unsup_gan_xp -TASK_DATA=/path/to/features/precompute_unfiltered_pca512_cls128_mean_pooled -TEXT_DATA=/path/to/data/phones # path to fairseq-preprocessed GAN data (phones dir) -KENLM_PATH=/path/to/data/phones/kenlm.phn.o4.bin # KenLM 4-gram phoneme language model (LM data = GAN data here) - -PYTHONPATH=$FAIRSEQ_ROOT PREFIX=$PREFIX fairseq-hydra-train \ - -m --config-dir config/gan \ - --config-name w2vu \ - task.data=${TASK_DATA} \ - task.text_data=${TEXT_DATA} \ - task.kenlm_path=${KENLM_PATH} \ - common.user_dir=${FAIRSEQ_ROOT}/examples/wav2vec/unsupervised \ - model.code_penalty=2,4 model.gradient_penalty=1.5,2.0 \ - model.smoothness_weight=0.5,0.75,1.0 'common.seed=range(0,5)' -``` - - -Once we find the best checkpoint (chosen using unsupervised metric that combined language model perplexity and vocabulary usage), we can use it to generate phone labels (or word labels with an appropriate kaldi WFST): - -```shell -python w2vu_generate.py --config-dir config/generate --config-name viterbi \ -fairseq.common.user_dir=${FAIRSEQ_ROOT}/examples/wav2vec/unsupervised \ -fairseq.task.data=/path/to/dir/with/features \ -fairseq.common_eval.path=/path/to/gan/checkpoint \ -fairseq.dataset.gen_subset=valid results_path=/where/to/save/transcriptions -``` - -The decoding without LM works best on the same adjacent-mean-pooled features that the gan was trained on, while decoding with LM works better on features before the adjacent timestep mean-pooling step (without the "_pooled" suffix). - -## Iterative self-training + Kaldi LM-decoding -After the GAN training provides a first unsupervised model, we can then progressively refine the quality of transcriptions using several iterations of semi-supervised learning. We perform two iterations: first, pseudo-label the training data with the unsupervised GAN model and train an HMM on the pseudo-labels. Second, we relabel the training data with the HMM and then fine-tune the original wav2vec 2.0 model using the HMM pseudo-labels with a CTC loss. Note that HMM models use phonemes as output, while wav2vec 2.0 use letter. Both are decoded using WFST decoders into words. - - -Please see [this README](kaldi_self_train/README.md) for more instructions on how to do iterative self-training + Kaldi LM-decoding. - -*** Note: these instructions are a work in progress and will be updated over the next few days diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/config/finetuning/w2v_finetune.yaml b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/config/finetuning/w2v_finetune.yaml deleted file mode 100644 index 19a3ef34849d33836ec4084dcbd755c53879ba28..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/config/finetuning/w2v_finetune.yaml +++ /dev/null @@ -1,62 +0,0 @@ -# @package _group_ - -common: - fp16: true - log_format: json - log_interval: 200 - tensorboard_logdir: tb - -checkpoint: - no_epoch_checkpoints: true - save_interval_updates: 20000 - -task: - _name: audio_finetuning - data: ??? - normalize: true - labels: ltr - -dataset: - num_workers: 6 - max_tokens: 800000 - skip_invalid_size_inputs_valid_test: true - train_subset: train - valid_subset: valid - -distributed_training: - ddp_backend: legacy_ddp - distributed_world_size: 8 - find_unused_parameters: True - -criterion: - _name: ctc - zero_infinity: true - post_process: letter - -optimization: - max_update: 80000 - lr: [0.00003] - sentence_avg: true - update_freq: [1] - -optimizer: - _name: adam - adam_betas: (0.9,0.98) - adam_eps: 1e-08 - -lr_scheduler: - _name: tri_stage - phase_ratio: [0.1, 0.4, 0.5] - final_lr_scale: 0.05 - -model: - _name: wav2vec_ctc - w2v_path: ??? - apply_mask: true - mask_prob: 0.25 - mask_channel_prob: 0.1 - mask_channel_length: 64 - layerdrop: 0.1 - activation_dropout: 0.1 - feature_grad_mult: 0.0 - freeze_finetune_updates: 0 diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/config/gan/w2vu.yaml b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/config/gan/w2vu.yaml deleted file mode 100644 index 74f1829d1497560f6e1e006073f19716d36bc947..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/config/gan/w2vu.yaml +++ /dev/null @@ -1,115 +0,0 @@ -# @package _group_ - -common: - fp16: false - fp16_no_flatten_grads: true - log_format: json - log_interval: 100 - tensorboard_logdir: tb - reset_logging: false - suppress_crashes: false - -checkpoint: - save_interval: 1000 - save_interval_updates: 1000 - no_epoch_checkpoints: true - best_checkpoint_metric: weighted_lm_ppl - save_dir: . - -distributed_training: - distributed_world_size: 1 - -task: - _name: unpaired_audio_text - data: ??? - text_data: ??? - labels: phn - sort_by_length: false - unfiltered: false - max_length: null - append_eos: false - kenlm_path: ??? - -dataset: - num_workers: 6 - batch_size: 160 - skip_invalid_size_inputs_valid_test: true - valid_subset: valid - validate_interval: 1000 - validate_interval_updates: 1000 - -criterion: - _name: model - log_keys: - - accuracy_dense - - accuracy_token - - temp - - code_ppl - -optimization: - max_update: 150000 - clip_norm: 5.0 - lr: [0] - -optimizer: - _name: composite - groups: - generator: - lr: [0.0004] - lr_float: null - optimizer: - _name: adam - adam_betas: [0.5,0.98] - adam_eps: 1e-06 - weight_decay: 0 - amsgrad: false - lr_scheduler: - _name: fixed - warmup_updates: 0 - discriminator: - lr: [ 0.0005 ] - lr_float: null - optimizer: - _name: adam - adam_betas: [0.5,0.98] - adam_eps: 1e-06 - weight_decay: 0.0001 - amsgrad: false - lr_scheduler: - _name: fixed - warmup_updates: 0 - -lr_scheduler: pass_through - -model: - _name: wav2vec_u - - discriminator_dim: 384 - discriminator_depth: 2 - discriminator_kernel: 6 - discriminator_linear_emb: false - discriminator_causal: true - discriminator_max_pool: false - discriminator_act_after_linear: false - discriminator_dropout: 0.0 - discriminator_weight_norm: false - - generator_stride: 1 - generator_kernel: 4 - generator_bias: false - generator_dropout: 0.1 - - smoothness_weight: 0.5 - smoothing: 0 - smoothing_one_sided: false - gumbel: false - hard_gumbel: false - gradient_penalty: 1.5 - code_penalty: 4.0 - temp: [ 2,0.1,0.99995 ] - input_dim: 512 - - segmentation: - type: JOIN - mean_pool_join: false - remove_zeros: false diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/config/generate/viterbi.yaml b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/config/generate/viterbi.yaml deleted file mode 100644 index 9c88beebcb15f9047195c8c7e79c21eac59db418..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/config/generate/viterbi.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# @package _group_ - -fairseq: - task: - _name: unpaired_audio_text - labels: phn - data: ??? - sort_by_length: false - shuffle: false - text_data: '' - - common_eval: - path: ??? - quiet: true - - dataset: - gen_subset: valid - batch_size: 1 - -w2l_decoder: VITERBI -post_process: silence diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/config/timit_matched/test.uid b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/config/timit_matched/test.uid deleted file mode 100644 index 401008246a1bc2cbf309d9d0aa56710f0ff643bc..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/config/timit_matched/test.uid +++ /dev/null @@ -1,192 +0,0 @@ -FDHC0_SI1559 -FDHC0_SI2189 -FDHC0_SI929 -FDHC0_SX119 -FDHC0_SX209 -FDHC0_SX29 -FDHC0_SX299 -FDHC0_SX389 -FELC0_SI1386 -FELC0_SI2016 -FELC0_SI756 -FELC0_SX126 -FELC0_SX216 -FELC0_SX306 -FELC0_SX36 -FELC0_SX396 -FJLM0_SI1043 -FJLM0_SI1673 -FJLM0_SI2303 -FJLM0_SX143 -FJLM0_SX233 -FJLM0_SX323 -FJLM0_SX413 -FJLM0_SX53 -FMGD0_SI1564 -FMGD0_SI2194 -FMGD0_SI934 -FMGD0_SX124 -FMGD0_SX214 -FMGD0_SX304 -FMGD0_SX34 -FMGD0_SX394 -FMLD0_SI2185 -FMLD0_SI822 -FMLD0_SI925 -FMLD0_SX115 -FMLD0_SX205 -FMLD0_SX25 -FMLD0_SX295 -FMLD0_SX385 -FNLP0_SI1308 -FNLP0_SI1938 -FNLP0_SI678 -FNLP0_SX138 -FNLP0_SX228 -FNLP0_SX318 -FNLP0_SX408 -FNLP0_SX48 -FPAS0_SI1272 -FPAS0_SI2204 -FPAS0_SI944 -FPAS0_SX134 -FPAS0_SX224 -FPAS0_SX314 -FPAS0_SX404 -FPAS0_SX44 -FPKT0_SI1538 -FPKT0_SI2168 -FPKT0_SI908 -FPKT0_SX188 -FPKT0_SX278 -FPKT0_SX368 -FPKT0_SX8 -FPKT0_SX98 -MBPM0_SI1577 -MBPM0_SI1584 -MBPM0_SI947 -MBPM0_SX137 -MBPM0_SX227 -MBPM0_SX317 -MBPM0_SX407 -MBPM0_SX47 -MCMJ0_SI1094 -MCMJ0_SI464 -MCMJ0_SI602 -MCMJ0_SX104 -MCMJ0_SX14 -MCMJ0_SX194 -MCMJ0_SX284 -MCMJ0_SX374 -MDAB0_SI1039 -MDAB0_SI1669 -MDAB0_SI2299 -MDAB0_SX139 -MDAB0_SX229 -MDAB0_SX319 -MDAB0_SX409 -MDAB0_SX49 -MGRT0_SI1450 -MGRT0_SI2080 -MGRT0_SI820 -MGRT0_SX10 -MGRT0_SX100 -MGRT0_SX190 -MGRT0_SX280 -MGRT0_SX370 -MJDH0_SI1354 -MJDH0_SI1984 -MJDH0_SI724 -MJDH0_SX184 -MJDH0_SX274 -MJDH0_SX364 -MJDH0_SX4 -MJDH0_SX94 -MJLN0_SI1449 -MJLN0_SI2079 -MJLN0_SI819 -MJLN0_SX189 -MJLN0_SX279 -MJLN0_SX369 -MJLN0_SX9 -MJLN0_SX99 -MJMP0_SI1535 -MJMP0_SI1791 -MJMP0_SI905 -MJMP0_SX185 -MJMP0_SX275 -MJMP0_SX365 -MJMP0_SX5 -MJMP0_SX95 -MKLT0_SI1213 -MKLT0_SI1843 -MKLT0_SI583 -MKLT0_SX133 -MKLT0_SX223 -MKLT0_SX313 -MKLT0_SX403 -MKLT0_SX43 -MLLL0_SI1363 -MLLL0_SI1993 -MLLL0_SI733 -MLLL0_SX103 -MLLL0_SX13 -MLLL0_SX193 -MLLL0_SX283 -MLLL0_SX373 -MLNT0_SI1574 -MLNT0_SI1902 -MLNT0_SI642 -MLNT0_SX102 -MLNT0_SX12 -MLNT0_SX192 -MLNT0_SX282 -MLNT0_SX372 -MNJM0_SI1580 -MNJM0_SI2210 -MNJM0_SI950 -MNJM0_SX140 -MNJM0_SX230 -MNJM0_SX320 -MNJM0_SX410 -MNJM0_SX50 -MPAM0_SI1189 -MPAM0_SI1819 -MPAM0_SI1961 -MPAM0_SX109 -MPAM0_SX19 -MPAM0_SX199 -MPAM0_SX289 -MPAM0_SX379 -MTAS1_SI1473 -MTAS1_SI2098 -MTAS1_SI838 -MTAS1_SX118 -MTAS1_SX208 -MTAS1_SX28 -MTAS1_SX298 -MTAS1_SX388 -MTLS0_SI1370 -MTLS0_SI2000 -MTLS0_SI740 -MTLS0_SX110 -MTLS0_SX20 -MTLS0_SX200 -MTLS0_SX290 -MTLS0_SX380 -MWBT0_SI1553 -MWBT0_SI2183 -MWBT0_SI923 -MWBT0_SX113 -MWBT0_SX203 -MWBT0_SX23 -MWBT0_SX293 -MWBT0_SX383 -MWEW0_SI1361 -MWEW0_SI1991 -MWEW0_SI731 -MWEW0_SX101 -MWEW0_SX11 -MWEW0_SX191 -MWEW0_SX281 -MWEW0_SX371 diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/config/timit_matched/train.uid b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/config/timit_matched/train.uid deleted file mode 100644 index c39fd0b91d51e0ae15caf1e9701d0d9ef51ee21b..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/config/timit_matched/train.uid +++ /dev/null @@ -1,3696 +0,0 @@ -FAEM0_SI1392 -FAEM0_SI2022 -FAEM0_SI762 -FAEM0_SX132 -FAEM0_SX222 -FAEM0_SX312 -FAEM0_SX402 -FAEM0_SX42 -FAJW0_SI1263 -FAJW0_SI1893 -FAJW0_SI633 -FAJW0_SX183 -FAJW0_SX273 -FAJW0_SX3 -FAJW0_SX363 -FAJW0_SX93 -FALK0_SI1086 -FALK0_SI456 -FALK0_SI658 -FALK0_SX186 -FALK0_SX276 -FALK0_SX366 -FALK0_SX6 -FALK0_SX96 -FALR0_SI1325 -FALR0_SI1955 -FALR0_SI695 -FALR0_SX155 -FALR0_SX245 -FALR0_SX335 -FALR0_SX425 -FALR0_SX65 -FAPB0_SI1063 -FAPB0_SI1693 -FAPB0_SI2323 -FAPB0_SX163 -FAPB0_SX253 -FAPB0_SX343 -FAPB0_SX433 -FAPB0_SX73 -FBAS0_SI1387 -FBAS0_SI1472 -FBAS0_SI2066 -FBAS0_SX127 -FBAS0_SX217 -FBAS0_SX307 -FBAS0_SX37 -FBAS0_SX397 -FBCG1_SI1612 -FBCG1_SI2242 -FBCG1_SI982 -FBCG1_SX172 -FBCG1_SX262 -FBCG1_SX352 -FBCG1_SX442 -FBCG1_SX82 -FBCH0_SI1586 -FBCH0_SI956 -FBCH0_SI959 -FBCH0_SX146 -FBCH0_SX236 -FBCH0_SX326 -FBCH0_SX416 -FBCH0_SX56 -FBJL0_SI1552 -FBJL0_SI2182 -FBJL0_SI922 -FBJL0_SX112 -FBJL0_SX202 -FBJL0_SX22 -FBJL0_SX292 -FBJL0_SX382 -FBLV0_SI1058 -FBLV0_SI1688 -FBLV0_SI2318 -FBLV0_SX158 -FBLV0_SX248 -FBLV0_SX338 -FBLV0_SX428 -FBLV0_SX68 -FBMH0_SI1136 -FBMH0_SI1766 -FBMH0_SI970 -FBMH0_SX146 -FBMH0_SX236 -FBMH0_SX326 -FBMH0_SX416 -FBMH0_SX56 -FBMJ0_SI1776 -FBMJ0_SI516 -FBMJ0_SI815 -FBMJ0_SX156 -FBMJ0_SX246 -FBMJ0_SX336 -FBMJ0_SX426 -FBMJ0_SX66 -FCAG0_SI1503 -FCAG0_SI1641 -FCAG0_SI2133 -FCAG0_SX153 -FCAG0_SX243 -FCAG0_SX333 -FCAG0_SX423 -FCAG0_SX63 -FCAJ0_SI1479 -FCAJ0_SI1804 -FCAJ0_SI849 -FCAJ0_SX129 -FCAJ0_SX219 -FCAJ0_SX309 -FCAJ0_SX39 -FCAJ0_SX399 -FCDR1_SI1186 -FCDR1_SI1816 -FCDR1_SI556 -FCDR1_SX106 -FCDR1_SX16 -FCDR1_SX196 -FCDR1_SX286 -FCDR1_SX376 -FCEG0_SI1248 -FCEG0_SI1878 -FCEG0_SI618 -FCEG0_SX168 -FCEG0_SX258 -FCEG0_SX348 -FCEG0_SX438 -FCEG0_SX78 -FCJF0_SI1027 -FCJF0_SI1657 -FCJF0_SI648 -FCJF0_SX127 -FCJF0_SX217 -FCJF0_SX307 -FCJF0_SX37 -FCJF0_SX397 -FCJS0_SI1607 -FCJS0_SI2237 -FCJS0_SI977 -FCJS0_SX167 -FCJS0_SX257 -FCJS0_SX347 -FCJS0_SX437 -FCJS0_SX77 -FCKE0_SI1111 -FCKE0_SI1741 -FCKE0_SI481 -FCKE0_SX121 -FCKE0_SX211 -FCKE0_SX301 -FCKE0_SX31 -FCKE0_SX391 -FCLT0_SI1438 -FCLT0_SI2068 -FCLT0_SI808 -FCLT0_SX178 -FCLT0_SX268 -FCLT0_SX358 -FCLT0_SX448 -FCLT0_SX88 -FCMG0_SI1142 -FCMG0_SI1242 -FCMG0_SI1872 -FCMG0_SX162 -FCMG0_SX252 -FCMG0_SX342 -FCMG0_SX432 -FCMG0_SX72 -FCMM0_SI1083 -FCMM0_SI1957 -FCMM0_SI453 -FCMM0_SX183 -FCMM0_SX273 -FCMM0_SX363 -FCMM0_SX420 -FCMM0_SX93 -FCRZ0_SI1913 -FCRZ0_SI2053 -FCRZ0_SI793 -FCRZ0_SX163 -FCRZ0_SX253 -FCRZ0_SX343 -FCRZ0_SX433 -FCRZ0_SX73 -FCYL0_SI1297 -FCYL0_SI1927 -FCYL0_SI667 -FCYL0_SX127 -FCYL0_SX217 -FCYL0_SX349 -FCYL0_SX37 -FCYL0_SX397 -FDAS1_SI1461 -FDAS1_SI2091 -FDAS1_SI831 -FDAS1_SX111 -FDAS1_SX201 -FDAS1_SX21 -FDAS1_SX291 -FDAS1_SX381 -FDAW0_SI1271 -FDAW0_SI1406 -FDAW0_SI2036 -FDAW0_SX146 -FDAW0_SX236 -FDAW0_SX326 -FDAW0_SX416 -FDAW0_SX56 -FDFB0_SI1318 -FDFB0_SI1948 -FDFB0_SI2010 -FDFB0_SX148 -FDFB0_SX238 -FDFB0_SX328 -FDFB0_SX418 -FDFB0_SX58 -FDJH0_SI1565 -FDJH0_SI2195 -FDJH0_SI935 -FDJH0_SX125 -FDJH0_SX215 -FDJH0_SX305 -FDJH0_SX35 -FDJH0_SX395 -FDKN0_SI1081 -FDKN0_SI1202 -FDKN0_SI1711 -FDKN0_SX181 -FDKN0_SX271 -FDKN0_SX361 -FDKN0_SX451 -FDKN0_SX91 -FDML0_SI1149 -FDML0_SI1779 -FDML0_SI2075 -FDML0_SX159 -FDML0_SX249 -FDML0_SX339 -FDML0_SX429 -FDML0_SX69 -FDMY0_SI1197 -FDMY0_SI567 -FDMY0_SI714 -FDMY0_SX117 -FDMY0_SX207 -FDMY0_SX27 -FDMY0_SX297 -FDMY0_SX387 -FDNC0_SI1278 -FDNC0_SI1908 -FDNC0_SI2287 -FDNC0_SX108 -FDNC0_SX18 -FDNC0_SX198 -FDNC0_SX288 -FDNC0_SX378 -FDTD0_SI1561 -FDTD0_SI2191 -FDTD0_SI931 -FDTD0_SX121 -FDTD0_SX211 -FDTD0_SX301 -FDTD0_SX321 -FDTD0_SX391 -FDXW0_SI1511 -FDXW0_SI2141 -FDXW0_SI881 -FDXW0_SX161 -FDXW0_SX251 -FDXW0_SX341 -FDXW0_SX431 -FDXW0_SX71 -FEAC0_SI1245 -FEAC0_SI1875 -FEAC0_SI615 -FEAC0_SX165 -FEAC0_SX255 -FEAC0_SX345 -FEAC0_SX435 -FEAC0_SX75 -FEAR0_SI1252 -FEAR0_SI1882 -FEAR0_SI622 -FEAR0_SX172 -FEAR0_SX262 -FEAR0_SX352 -FEAR0_SX442 -FEAR0_SX82 -FECD0_SI1418 -FECD0_SI2048 -FECD0_SI788 -FECD0_SX158 -FECD0_SX248 -FECD0_SX338 -FECD0_SX428 -FECD0_SX68 -FEEH0_SI1112 -FEEH0_SI1742 -FEEH0_SI471 -FEEH0_SX122 -FEEH0_SX212 -FEEH0_SX302 -FEEH0_SX32 -FEEH0_SX392 -FEME0_SI1505 -FEME0_SI2135 -FEME0_SI875 -FEME0_SX155 -FEME0_SX245 -FEME0_SX335 -FEME0_SX425 -FEME0_SX65 -FETB0_SI1148 -FETB0_SI1778 -FETB0_SI518 -FETB0_SX158 -FETB0_SX248 -FETB0_SX338 -FETB0_SX428 -FETB0_SX68 -FEXM0_SI1101 -FEXM0_SI1731 -FEXM0_SI482 -FEXM0_SX111 -FEXM0_SX201 -FEXM0_SX291 -FEXM0_SX366 -FEXM0_SX381 -FGCS0_SI1486 -FGCS0_SI2116 -FGCS0_SI856 -FGCS0_SX136 -FGCS0_SX226 -FGCS0_SX316 -FGCS0_SX406 -FGCS0_SX46 -FGDP0_SI1618 -FGDP0_SI2248 -FGDP0_SI988 -FGDP0_SX178 -FGDP0_SX268 -FGDP0_SX358 -FGDP0_SX448 -FGDP0_SX88 -FGMB0_SI1145 -FGMB0_SI1775 -FGMB0_SI515 -FGMB0_SX155 -FGMB0_SX245 -FGMB0_SX335 -FGMB0_SX425 -FGMB0_SX65 -FGRW0_SI1152 -FGRW0_SI1782 -FGRW0_SI1990 -FGRW0_SX162 -FGRW0_SX252 -FGRW0_SX342 -FGRW0_SX432 -FGRW0_SX72 -FHLM0_SI1560 -FHLM0_SI2190 -FHLM0_SI930 -FHLM0_SX120 -FHLM0_SX210 -FHLM0_SX300 -FHLM0_SX349 -FHLM0_SX390 -FHXS0_SI1075 -FHXS0_SI2302 -FHXS0_SI2335 -FHXS0_SX175 -FHXS0_SX265 -FHXS0_SX355 -FHXS0_SX445 -FHXS0_SX85 -FJDM2_SI1582 -FJDM2_SI1964 -FJDM2_SI2212 -FJDM2_SX142 -FJDM2_SX232 -FJDM2_SX322 -FJDM2_SX412 -FJDM2_SX52 -FJEN0_SI1047 -FJEN0_SI1677 -FJEN0_SI2307 -FJEN0_SX147 -FJEN0_SX237 -FJEN0_SX327 -FJEN0_SX417 -FJEN0_SX57 -FJHK0_SI1022 -FJHK0_SI1652 -FJHK0_SI2282 -FJHK0_SX122 -FJHK0_SX212 -FJHK0_SX302 -FJHK0_SX32 -FJHK0_SX392 -FJKL0_SI1562 -FJKL0_SI2192 -FJKL0_SI932 -FJKL0_SX122 -FJKL0_SX212 -FJKL0_SX302 -FJKL0_SX32 -FJKL0_SX392 -FJLG0_SI1506 -FJLG0_SI1889 -FJLG0_SI2306 -FJLG0_SX179 -FJLG0_SX269 -FJLG0_SX359 -FJLG0_SX449 -FJLG0_SX89 -FJLR0_SI1231 -FJLR0_SI1861 -FJLR0_SI601 -FJLR0_SX151 -FJLR0_SX241 -FJLR0_SX331 -FJLR0_SX421 -FJLR0_SX61 -FJRB0_SI1302 -FJRB0_SI1932 -FJRB0_SI672 -FJRB0_SX132 -FJRB0_SX222 -FJRB0_SX312 -FJRB0_SX402 -FJRB0_SX42 -FJRP1_SI1432 -FJRP1_SI2062 -FJRP1_SI802 -FJRP1_SX172 -FJRP1_SX262 -FJRP1_SX352 -FJRP1_SX442 -FJRP1_SX82 -FJSK0_SI1052 -FJSK0_SI1682 -FJSK0_SI2312 -FJSK0_SX152 -FJSK0_SX242 -FJSK0_SX332 -FJSK0_SX422 -FJSK0_SX62 -FJSP0_SI1434 -FJSP0_SI1763 -FJSP0_SI804 -FJSP0_SX174 -FJSP0_SX264 -FJSP0_SX354 -FJSP0_SX444 -FJSP0_SX84 -FJWB1_SI2055 -FJWB1_SI748 -FJWB1_SI795 -FJWB1_SX165 -FJWB1_SX255 -FJWB1_SX345 -FJWB1_SX435 -FJWB1_SX75 -FJXM0_SI1211 -FJXM0_SI1971 -FJXM0_SI581 -FJXM0_SX131 -FJXM0_SX221 -FJXM0_SX311 -FJXM0_SX401 -FJXM0_SX41 -FJXP0_SI1122 -FJXP0_SI1752 -FJXP0_SI492 -FJXP0_SX132 -FJXP0_SX222 -FJXP0_SX312 -FJXP0_SX402 -FJXP0_SX42 -FKAA0_SI1208 -FKAA0_SI1838 -FKAA0_SI578 -FKAA0_SX128 -FKAA0_SX218 -FKAA0_SX308 -FKAA0_SX38 -FKAA0_SX398 -FKDE0_SI1141 -FKDE0_SI1771 -FKDE0_SI2221 -FKDE0_SX151 -FKDE0_SX241 -FKDE0_SX331 -FKDE0_SX421 -FKDE0_SX61 -FKDW0_SI1207 -FKDW0_SI1891 -FKDW0_SI577 -FKDW0_SX127 -FKDW0_SX217 -FKDW0_SX307 -FKDW0_SX37 -FKDW0_SX397 -FKFB0_SI1608 -FKFB0_SI2238 -FKFB0_SI978 -FKFB0_SX168 -FKFB0_SX258 -FKFB0_SX348 -FKFB0_SX438 -FKFB0_SX78 -FKKH0_SI1290 -FKKH0_SI1920 -FKKH0_SI660 -FKKH0_SX120 -FKKH0_SX210 -FKKH0_SX30 -FKKH0_SX300 -FKKH0_SX390 -FKLC0_SI1615 -FKLC0_SI2245 -FKLC0_SI985 -FKLC0_SX175 -FKLC0_SX265 -FKLC0_SX355 -FKLC0_SX445 -FKLC0_SX85 -FKLC1_SI1048 -FKLC1_SI1678 -FKLC1_SI2308 -FKLC1_SX148 -FKLC1_SX238 -FKLC1_SX328 -FKLC1_SX418 -FKLC1_SX58 -FKLH0_SI1257 -FKLH0_SI1887 -FKLH0_SI627 -FKLH0_SX177 -FKLH0_SX267 -FKLH0_SX357 -FKLH0_SX447 -FKLH0_SX87 -FKSR0_SI1117 -FKSR0_SI1747 -FKSR0_SI487 -FKSR0_SX161 -FKSR0_SX217 -FKSR0_SX366 -FKSR0_SX37 -FKSR0_SX397 -FLAC0_SI1339 -FLAC0_SI2161 -FLAC0_SI901 -FLAC0_SX181 -FLAC0_SX271 -FLAC0_SX361 -FLAC0_SX451 -FLAC0_SX91 -FLAG0_SI1464 -FLAG0_SI2094 -FLAG0_SI834 -FLAG0_SX114 -FLAG0_SX204 -FLAG0_SX24 -FLAG0_SX294 -FLAG0_SX384 -FLEH0_SI1051 -FLEH0_SI1681 -FLEH0_SI2311 -FLEH0_SX151 -FLEH0_SX241 -FLEH0_SX331 -FLEH0_SX421 -FLEH0_SX61 -FLET0_SI1137 -FLET0_SI1767 -FLET0_SI507 -FLET0_SX147 -FLET0_SX237 -FLET0_SX277 -FLET0_SX417 -FLET0_SX57 -FLHD0_SI1344 -FLHD0_SI1827 -FLHD0_SI1974 -FLHD0_SX174 -FLHD0_SX264 -FLHD0_SX354 -FLHD0_SX444 -FLHD0_SX84 -FLJA0_SI1078 -FLJA0_SI1708 -FLJA0_SI2338 -FLJA0_SX178 -FLJA0_SX268 -FLJA0_SX358 -FLJA0_SX448 -FLJA0_SX88 -FLJD0_SI1516 -FLJD0_SI2146 -FLJD0_SI886 -FLJD0_SX166 -FLJD0_SX256 -FLJD0_SX346 -FLJD0_SX436 -FLJD0_SX76 -FLJG0_SI1611 -FLJG0_SI2241 -FLJG0_SI981 -FLJG0_SX171 -FLJG0_SX261 -FLJG0_SX351 -FLJG0_SX441 -FLJG0_SX81 -FLKM0_SI1880 -FLKM0_SI620 -FLKM0_SI686 -FLKM0_SX116 -FLKM0_SX260 -FLKM0_SX350 -FLKM0_SX440 -FLKM0_SX80 -FLMA0_SI1243 -FLMA0_SI1873 -FLMA0_SI613 -FLMA0_SX163 -FLMA0_SX253 -FLMA0_SX343 -FLMA0_SX433 -FLMA0_SX73 -FLMC0_SI1372 -FLMC0_SI2002 -FLMC0_SI742 -FLMC0_SX112 -FLMC0_SX22 -FLMC0_SX292 -FLMC0_SX336 -FLMC0_SX382 -FLMK0_SI1035 -FLMK0_SI1229 -FLMK0_SI2295 -FLMK0_SX135 -FLMK0_SX225 -FLMK0_SX315 -FLMK0_SX405 -FLMK0_SX45 -FLOD0_SI1287 -FLOD0_SI1917 -FLOD0_SI657 -FLOD0_SX117 -FLOD0_SX171 -FLOD0_SX207 -FLOD0_SX297 -FLOD0_SX387 -FLTM0_SI1070 -FLTM0_SI1700 -FLTM0_SI2330 -FLTM0_SX170 -FLTM0_SX260 -FLTM0_SX350 -FLTM0_SX440 -FLTM0_SX80 -FMAH1_SI1509 -FMAH1_SI2139 -FMAH1_SI879 -FMAH1_SX159 -FMAH1_SX249 -FMAH1_SX339 -FMAH1_SX429 -FMAH1_SX69 -FMBG0_SI1160 -FMBG0_SI1790 -FMBG0_SI2264 -FMBG0_SX260 -FMBG0_SX3 -FMBG0_SX350 -FMBG0_SX440 -FMBG0_SX80 -FMEM0_SI1377 -FMEM0_SI2007 -FMEM0_SI747 -FMEM0_SX117 -FMEM0_SX207 -FMEM0_SX297 -FMEM0_SX333 -FMEM0_SX387 -FMJB0_SI1177 -FMJB0_SI1807 -FMJB0_SI547 -FMJB0_SX187 -FMJB0_SX277 -FMJB0_SX367 -FMJB0_SX7 -FMJB0_SX97 -FMJF0_SI1254 -FMJF0_SI1884 -FMJF0_SI624 -FMJF0_SX174 -FMJF0_SX264 -FMJF0_SX354 -FMJF0_SX444 -FMJF0_SX84 -FMJU0_SI1389 -FMJU0_SI2019 -FMJU0_SI759 -FMJU0_SX129 -FMJU0_SX219 -FMJU0_SX309 -FMJU0_SX39 -FMJU0_SX399 -FMKC0_SI1041 -FMKC0_SI1072 -FMKC0_SI1702 -FMKC0_SX172 -FMKC0_SX262 -FMKC0_SX352 -FMKC0_SX442 -FMKC0_SX82 -FMKF0_SI1018 -FMKF0_SI1536 -FMKF0_SI906 -FMKF0_SX186 -FMKF0_SX276 -FMKF0_SX366 -FMKF0_SX6 -FMKF0_SX96 -FMMH0_SI1537 -FMMH0_SI2167 -FMMH0_SI907 -FMMH0_SX187 -FMMH0_SX367 -FMMH0_SX420 -FMMH0_SX7 -FMMH0_SX97 -FMPG0_SI1602 -FMPG0_SI2232 -FMPG0_SI972 -FMPG0_SX162 -FMPG0_SX252 -FMPG0_SX342 -FMPG0_SX432 -FMPG0_SX72 -FNKL0_SI1522 -FNKL0_SI2152 -FNKL0_SI892 -FNKL0_SX172 -FNKL0_SX196 -FNKL0_SX262 -FNKL0_SX442 -FNKL0_SX82 -FNTB0_SI1203 -FNTB0_SI573 -FNTB0_SI679 -FNTB0_SX123 -FNTB0_SX213 -FNTB0_SX303 -FNTB0_SX33 -FNTB0_SX393 -FPAB1_SI1471 -FPAB1_SI2101 -FPAB1_SI841 -FPAB1_SX121 -FPAB1_SX211 -FPAB1_SX301 -FPAB1_SX31 -FPAB1_SX391 -FPAC0_SI1921 -FPAC0_SI2011 -FPAC0_SI661 -FPAC0_SX121 -FPAC0_SX211 -FPAC0_SX301 -FPAC0_SX31 -FPAC0_SX391 -FPAD0_SI1346 -FPAD0_SI1976 -FPAD0_SI716 -FPAD0_SX176 -FPAD0_SX266 -FPAD0_SX356 -FPAD0_SX446 -FPAD0_SX86 -FPAF0_SI1054 -FPAF0_SI1684 -FPAF0_SI2314 -FPAF0_SX154 -FPAF0_SX244 -FPAF0_SX334 -FPAF0_SX424 -FPAF0_SX64 -FPAZ0_SI1593 -FPAZ0_SI2223 -FPAZ0_SI963 -FPAZ0_SX153 -FPAZ0_SX243 -FPAZ0_SX27 -FPAZ0_SX423 -FPAZ0_SX63 -FPJF0_SI1046 -FPJF0_SI1259 -FPJF0_SI1676 -FPJF0_SX146 -FPJF0_SX236 -FPJF0_SX326 -FPJF0_SX352 -FPJF0_SX56 -FPLS0_SI1590 -FPLS0_SI2220 -FPLS0_SI960 -FPLS0_SX150 -FPLS0_SX240 -FPLS0_SX3 -FPLS0_SX330 -FPLS0_SX60 -FPMY0_SI1153 -FPMY0_SI1783 -FPMY0_SI523 -FPMY0_SX163 -FPMY0_SX196 -FPMY0_SX253 -FPMY0_SX343 -FPMY0_SX73 -FREH0_SI1315 -FREH0_SI1945 -FREH0_SI685 -FREH0_SX145 -FREH0_SX235 -FREH0_SX325 -FREH0_SX415 -FREH0_SX55 -FRJB0_SI1427 -FRJB0_SI1470 -FRJB0_SI1794 -FRJB0_SX167 -FRJB0_SX257 -FRJB0_SX347 -FRJB0_SX437 -FRJB0_SX77 -FRLL0_SI1514 -FRLL0_SI805 -FRLL0_SI884 -FRLL0_SX164 -FRLL0_SX254 -FRLL0_SX344 -FRLL0_SX434 -FRLL0_SX74 -FSAG0_SI1323 -FSAG0_SI1953 -FSAG0_SI693 -FSAG0_SX153 -FSAG0_SX243 -FSAG0_SX333 -FSAG0_SX423 -FSAG0_SX63 -FSAH0_SI1244 -FSAH0_SI1874 -FSAH0_SI614 -FSAH0_SX164 -FSAH0_SX327 -FSAH0_SX344 -FSAH0_SX434 -FSAH0_SX74 -FSAK0_SI1300 -FSAK0_SI1930 -FSAK0_SI670 -FSAK0_SX130 -FSAK0_SX220 -FSAK0_SX310 -FSAK0_SX40 -FSAK0_SX400 -FSBK0_SI1069 -FSBK0_SI1699 -FSBK0_SI2329 -FSBK0_SX169 -FSBK0_SX259 -FSBK0_SX349 -FSBK0_SX439 -FSBK0_SX79 -FSCN0_SI1886 -FSCN0_SI626 -FSCN0_SI705 -FSCN0_SX176 -FSCN0_SX266 -FSCN0_SX356 -FSCN0_SX446 -FSCN0_SX86 -FSDC0_SI1312 -FSDC0_SI1942 -FSDC0_SI2234 -FSDC0_SX142 -FSDC0_SX232 -FSDC0_SX322 -FSDC0_SX412 -FSDC0_SX52 -FSDJ0_SI1115 -FSDJ0_SI1745 -FSDJ0_SI485 -FSDJ0_SX125 -FSDJ0_SX215 -FSDJ0_SX305 -FSDJ0_SX35 -FSDJ0_SX395 -FSGF0_SI1557 -FSGF0_SI2187 -FSGF0_SI927 -FSGF0_SX117 -FSGF0_SX207 -FSGF0_SX27 -FSGF0_SX297 -FSGF0_SX387 -FSJG0_SI1570 -FSJG0_SI2200 -FSJG0_SI940 -FSJG0_SX130 -FSJG0_SX220 -FSJG0_SX310 -FSJG0_SX40 -FSJG0_SX400 -FSJK1_SI1025 -FSJK1_SI2285 -FSJK1_SI696 -FSJK1_SX125 -FSJK1_SX215 -FSJK1_SX305 -FSJK1_SX35 -FSJK1_SX395 -FSJS0_SI1171 -FSJS0_SI1801 -FSJS0_SI541 -FSJS0_SX181 -FSJS0_SX271 -FSJS0_SX361 -FSJS0_SX451 -FSJS0_SX91 -FSJW0_SI1333 -FSJW0_SI1963 -FSJW0_SI703 -FSJW0_SX163 -FSJW0_SX253 -FSJW0_SX343 -FSJW0_SX433 -FSJW0_SX73 -FSKC0_SI1416 -FSKC0_SI2046 -FSKC0_SI786 -FSKC0_SX156 -FSKC0_SX246 -FSKC0_SX336 -FSKC0_SX426 -FSKC0_SX66 -FSKL0_SI1529 -FSKL0_SI2159 -FSKL0_SI899 -FSKL0_SX179 -FSKL0_SX269 -FSKL0_SX359 -FSKL0_SX449 -FSKL0_SX89 -FSKP0_SI1098 -FSKP0_SI1728 -FSKP0_SI468 -FSKP0_SX108 -FSKP0_SX18 -FSKP0_SX198 -FSKP0_SX288 -FSKP0_SX378 -FSLS0_SI1056 -FSLS0_SI1686 -FSLS0_SI2316 -FSLS0_SX156 -FSLS0_SX202 -FSLS0_SX246 -FSLS0_SX426 -FSLS0_SX66 -FSMA0_SI1621 -FSMA0_SI2251 -FSMA0_SI991 -FSMA0_SX181 -FSMA0_SX271 -FSMA0_SX361 -FSMA0_SX451 -FSMA0_SX91 -FSMM0_SI1314 -FSMM0_SI1944 -FSMM0_SI684 -FSMM0_SX144 -FSMM0_SX234 -FSMM0_SX324 -FSMM0_SX414 -FSMM0_SX54 -FSMS1_SI1504 -FSMS1_SI2134 -FSMS1_SI874 -FSMS1_SX154 -FSMS1_SX244 -FSMS1_SX334 -FSMS1_SX347 -FSMS1_SX64 -FSPM0_SI1241 -FSPM0_SI1871 -FSPM0_SI611 -FSPM0_SX161 -FSPM0_SX251 -FSPM0_SX341 -FSPM0_SX431 -FSPM0_SX71 -FSRH0_SI1719 -FSRH0_SI1931 -FSRH0_SI671 -FSRH0_SX131 -FSRH0_SX221 -FSRH0_SX311 -FSRH0_SX401 -FSRH0_SX41 -FSSB0_SI1082 -FSSB0_SI1712 -FSSB0_SI2342 -FSSB0_SX182 -FSSB0_SX272 -FSSB0_SX362 -FSSB0_SX452 -FSSB0_SX92 -FTAJ0_SI1329 -FTAJ0_SI474 -FTAJ0_SI699 -FTAJ0_SX159 -FTAJ0_SX249 -FTAJ0_SX339 -FTAJ0_SX429 -FTAJ0_SX69 -FTBR0_SI1402 -FTBR0_SI2181 -FTBR0_SI921 -FTBR0_SX111 -FTBR0_SX201 -FTBR0_SX21 -FTBR0_SX291 -FTBR0_SX381 -FTBW0_SI1345 -FTBW0_SI1975 -FTBW0_SI715 -FTBW0_SX175 -FTBW0_SX265 -FTBW0_SX355 -FTBW0_SX445 -FTBW0_SX85 -FTLG0_SI1743 -FTLG0_SI483 -FTLG0_SI840 -FTLG0_SX123 -FTLG0_SX213 -FTLG0_SX303 -FTLG0_SX33 -FTLG0_SX393 -FTMG0_SI1532 -FTMG0_SI2162 -FTMG0_SI902 -FTMG0_SX182 -FTMG0_SX272 -FTMG0_SX362 -FTMG0_SX452 -FTMG0_SX92 -FVFB0_SI1032 -FVFB0_SI1510 -FVFB0_SI2292 -FVFB0_SX132 -FVFB0_SX222 -FVFB0_SX312 -FVFB0_SX402 -FVFB0_SX42 -FVKB0_SI1159 -FVKB0_SI1789 -FVKB0_SI529 -FVKB0_SX169 -FVKB0_SX259 -FVKB0_SX349 -FVKB0_SX439 -FVKB0_SX79 -FVMH0_SI1466 -FVMH0_SI2096 -FVMH0_SI836 -FVMH0_SX116 -FVMH0_SX206 -FVMH0_SX26 -FVMH0_SX296 -FVMH0_SX386 -MABC0_SI1620 -MABC0_SI2041 -MABC0_SI781 -MABC0_SX151 -MABC0_SX241 -MABC0_SX331 -MABC0_SX421 -MABC0_SX61 -MADC0_SI1367 -MADC0_SI1997 -MADC0_SI737 -MADC0_SX107 -MADC0_SX17 -MADC0_SX197 -MADC0_SX287 -MADC0_SX377 -MADD0_SI1295 -MADD0_SI1798 -MADD0_SI538 -MADD0_SX178 -MADD0_SX268 -MADD0_SX358 -MADD0_SX448 -MADD0_SX88 -MAEB0_SI1411 -MAEB0_SI2250 -MAEB0_SI990 -MAEB0_SX180 -MAEB0_SX270 -MAEB0_SX360 -MAEB0_SX450 -MAEB0_SX90 -MAEO0_SI1326 -MAEO0_SI1655 -MAEO0_SI1956 -MAEO0_SX156 -MAEO0_SX246 -MAEO0_SX336 -MAEO0_SX426 -MAEO0_SX66 -MAFM0_SI1569 -MAFM0_SI2199 -MAFM0_SI939 -MAFM0_SX129 -MAFM0_SX219 -MAFM0_SX309 -MAFM0_SX39 -MAFM0_SX399 -MAJP0_SI1074 -MAJP0_SI1704 -MAJP0_SI2334 -MAJP0_SX174 -MAJP0_SX264 -MAJP0_SX354 -MAJP0_SX444 -MAJP0_SX84 -MAKB0_SI1016 -MAKB0_SI1646 -MAKB0_SI2276 -MAKB0_SX116 -MAKB0_SX206 -MAKB0_SX26 -MAKB0_SX296 -MAKB0_SX386 -MAKR0_SI1352 -MAKR0_SI1982 -MAKR0_SI722 -MAKR0_SX182 -MAKR0_SX272 -MAKR0_SX362 -MAKR0_SX452 -MAKR0_SX92 -MAPV0_SI1293 -MAPV0_SI1923 -MAPV0_SI663 -MAPV0_SX123 -MAPV0_SX213 -MAPV0_SX303 -MAPV0_SX33 -MAPV0_SX393 -MARC0_SI1188 -MARC0_SI1818 -MARC0_SI558 -MARC0_SX108 -MARC0_SX18 -MARC0_SX198 -MARC0_SX288 -MARC0_SX378 -MARW0_SI1276 -MARW0_SI1906 -MARW0_SI646 -MARW0_SX106 -MARW0_SX16 -MARW0_SX286 -MARW0_SX349 -MARW0_SX376 -MBAR0_SI1319 -MBAR0_SI1949 -MBAR0_SI689 -MBAR0_SX149 -MBAR0_SX239 -MBAR0_SX329 -MBAR0_SX419 -MBAR0_SX59 -MBBR0_SI1055 -MBBR0_SI1685 -MBBR0_SI2315 -MBBR0_SX155 -MBBR0_SX245 -MBBR0_SX335 -MBBR0_SX425 -MBBR0_SX65 -MBCG0_SI2217 -MBCG0_SI486 -MBCG0_SI957 -MBCG0_SX147 -MBCG0_SX237 -MBCG0_SX327 -MBCG0_SX417 -MBCG0_SX57 -MBEF0_SI1281 -MBEF0_SI1911 -MBEF0_SI651 -MBEF0_SX111 -MBEF0_SX201 -MBEF0_SX21 -MBEF0_SX291 -MBEF0_SX381 -MBGT0_SI1341 -MBGT0_SI1841 -MBGT0_SI711 -MBGT0_SX171 -MBGT0_SX261 -MBGT0_SX351 -MBGT0_SX441 -MBGT0_SX81 -MBJV0_SI1247 -MBJV0_SI1877 -MBJV0_SI617 -MBJV0_SX167 -MBJV0_SX257 -MBJV0_SX347 -MBJV0_SX437 -MBJV0_SX77 -MBMA0_SI1222 -MBMA0_SI1852 -MBMA0_SI592 -MBMA0_SX142 -MBMA0_SX232 -MBMA0_SX322 -MBMA0_SX412 -MBMA0_SX52 -MBMA1_SI2207 -MBMA1_SI2214 -MBMA1_SI954 -MBMA1_SX144 -MBMA1_SX234 -MBMA1_SX324 -MBMA1_SX414 -MBMA1_SX54 -MBML0_SI1169 -MBML0_SI1799 -MBML0_SI539 -MBML0_SX179 -MBML0_SX269 -MBML0_SX359 -MBML0_SX449 -MBML0_SX89 -MBOM0_SI1014 -MBOM0_SI1644 -MBOM0_SI2274 -MBOM0_SX114 -MBOM0_SX204 -MBOM0_SX294 -MBOM0_SX311 -MBOM0_SX384 -MBSB0_SI1353 -MBSB0_SI1983 -MBSB0_SI723 -MBSB0_SX183 -MBSB0_SX273 -MBSB0_SX3 -MBSB0_SX363 -MBSB0_SX93 -MBTH0_SI2102 -MBTH0_SI505 -MBTH0_SI757 -MBTH0_SX122 -MBTH0_SX212 -MBTH0_SX302 -MBTH0_SX32 -MBTH0_SX392 -MBWP0_SI1531 -MBWP0_SI1969 -MBWP0_SI709 -MBWP0_SX169 -MBWP0_SX259 -MBWP0_SX349 -MBWP0_SX439 -MBWP0_SX79 -MCAE0_SI1447 -MCAE0_SI2077 -MCAE0_SI817 -MCAE0_SX187 -MCAE0_SX277 -MCAE0_SX367 -MCAE0_SX7 -MCAE0_SX97 -MCAL0_SI1138 -MCAL0_SI1768 -MCAL0_SI508 -MCAL0_SX148 -MCAL0_SX238 -MCAL0_SX328 -MCAL0_SX418 -MCAL0_SX58 -MCDC0_SI1292 -MCDC0_SI1922 -MCDC0_SI662 -MCDC0_SX122 -MCDC0_SX212 -MCDC0_SX302 -MCDC0_SX32 -MCDC0_SX392 -MCDD0_SI1513 -MCDD0_SI2143 -MCDD0_SI883 -MCDD0_SX163 -MCDD0_SX253 -MCDD0_SX343 -MCDD0_SX433 -MCDD0_SX73 -MCDR0_SI1154 -MCDR0_SI1784 -MCDR0_SI524 -MCDR0_SX164 -MCDR0_SX254 -MCDR0_SX344 -MCDR0_SX434 -MCDR0_SX74 -MCEF0_SI1135 -MCEF0_SI1765 -MCEF0_SI842 -MCEF0_SX145 -MCEF0_SX235 -MCEF0_SX325 -MCEF0_SX415 -MCEF0_SX55 -MCEW0_SI1442 -MCEW0_SI2072 -MCEW0_SI812 -MCEW0_SX182 -MCEW0_SX272 -MCEW0_SX362 -MCEW0_SX452 -MCEW0_SX92 -MCHL0_SI1347 -MCHL0_SI1404 -MCHL0_SI1977 -MCHL0_SX177 -MCHL0_SX267 -MCHL0_SX357 -MCHL0_SX447 -MCHL0_SX87 -MCLK0_SI1660 -MCLK0_SI2290 -MCLK0_SI650 -MCLK0_SX130 -MCLK0_SX220 -MCLK0_SX310 -MCLK0_SX40 -MCLK0_SX400 -MCLM0_SI1456 -MCLM0_SI2086 -MCLM0_SI826 -MCLM0_SX106 -MCLM0_SX16 -MCLM0_SX196 -MCLM0_SX286 -MCLM0_SX376 -MCPM0_SI1194 -MCPM0_SI1824 -MCPM0_SI564 -MCPM0_SX114 -MCPM0_SX204 -MCPM0_SX24 -MCPM0_SX294 -MCPM0_SX384 -MCRE0_SI1121 -MCRE0_SI1725 -MCRE0_SI1751 -MCRE0_SX131 -MCRE0_SX221 -MCRE0_SX24 -MCRE0_SX401 -MCRE0_SX41 -MCSS0_SI1380 -MCSS0_SI688 -MCSS0_SI750 -MCSS0_SX120 -MCSS0_SX210 -MCSS0_SX30 -MCSS0_SX300 -MCSS0_SX390 -MCTH0_SI1209 -MCTH0_SI1839 -MCTH0_SI579 -MCTH0_SX129 -MCTH0_SX219 -MCTH0_SX309 -MCTH0_SX39 -MCTH0_SX399 -MCTM0_SI1350 -MCTM0_SI1980 -MCTM0_SI720 -MCTM0_SX180 -MCTM0_SX270 -MCTM0_SX360 -MCTM0_SX450 -MCTM0_SX90 -MCXM0_SI1351 -MCXM0_SI1981 -MCXM0_SI721 -MCXM0_SX181 -MCXM0_SX271 -MCXM0_SX361 -MCXM0_SX451 -MCXM0_SX91 -MDAC0_SI1261 -MDAC0_SI1837 -MDAC0_SI631 -MDAC0_SX181 -MDAC0_SX271 -MDAC0_SX361 -MDAC0_SX451 -MDAC0_SX91 -MDAS0_SI1266 -MDAS0_SI1896 -MDAS0_SI636 -MDAS0_SX186 -MDAS0_SX21 -MDAS0_SX276 -MDAS0_SX6 -MDAS0_SX96 -MDBB1_SI1006 -MDBB1_SI1636 -MDBB1_SI2056 -MDBB1_SX106 -MDBB1_SX16 -MDBB1_SX196 -MDBB1_SX286 -MDBB1_SX376 -MDBP0_SI1158 -MDBP0_SI1788 -MDBP0_SI528 -MDBP0_SX168 -MDBP0_SX258 -MDBP0_SX348 -MDBP0_SX438 -MDBP0_SX78 -MDCD0_SI1415 -MDCD0_SI2045 -MDCD0_SI785 -MDCD0_SX155 -MDCD0_SX245 -MDCD0_SX335 -MDCD0_SX425 -MDCD0_SX65 -MDCM0_SI1480 -MDCM0_SI2110 -MDCM0_SI850 -MDCM0_SX130 -MDCM0_SX220 -MDCM0_SX310 -MDCM0_SX40 -MDCM0_SX400 -MDDC0_SI1419 -MDDC0_SI2049 -MDDC0_SI789 -MDDC0_SX159 -MDDC0_SX249 -MDDC0_SX339 -MDDC0_SX429 -MDDC0_SX69 -MDED0_SI1170 -MDED0_SI1800 -MDED0_SI540 -MDED0_SX180 -MDED0_SX270 -MDED0_SX360 -MDED0_SX450 -MDED0_SX90 -MDEF0_SI1123 -MDEF0_SI1563 -MDEF0_SI2193 -MDEF0_SX123 -MDEF0_SX213 -MDEF0_SX303 -MDEF0_SX33 -MDEF0_SX393 -MDEM0_SI1868 -MDEM0_SI608 -MDEM0_SI800 -MDEM0_SX158 -MDEM0_SX248 -MDEM0_SX338 -MDEM0_SX428 -MDEM0_SX68 -MDHL0_SI1439 -MDHL0_SI2069 -MDHL0_SI809 -MDHL0_SX179 -MDHL0_SX269 -MDHL0_SX359 -MDHL0_SX449 -MDHL0_SX89 -MDHS0_SI1530 -MDHS0_SI2160 -MDHS0_SI900 -MDHS0_SX180 -MDHS0_SX270 -MDHS0_SX360 -MDHS0_SX450 -MDHS0_SX90 -MDJM0_SI1455 -MDJM0_SI2085 -MDJM0_SI825 -MDJM0_SX105 -MDJM0_SX15 -MDJM0_SX195 -MDJM0_SX285 -MDJM0_SX375 -MDKS0_SI1066 -MDKS0_SI1696 -MDKS0_SI2326 -MDKS0_SX166 -MDKS0_SX256 -MDKS0_SX346 -MDKS0_SX436 -MDKS0_SX76 -MDLB0_SI1306 -MDLB0_SI1936 -MDLB0_SI676 -MDLB0_SX136 -MDLB0_SX226 -MDLB0_SX316 -MDLB0_SX406 -MDLB0_SX46 -MDLC0_SI1395 -MDLC0_SI2025 -MDLC0_SI765 -MDLC0_SX135 -MDLC0_SX225 -MDLC0_SX315 -MDLC0_SX405 -MDLC0_SX45 -MDLC1_SI1435 -MDLC1_SI2065 -MDLC1_SI2144 -MDLC1_SX175 -MDLC1_SX265 -MDLC1_SX355 -MDLC1_SX445 -MDLC1_SX85 -MDLC2_SI1614 -MDLC2_SI2244 -MDLC2_SI984 -MDLC2_SX174 -MDLC2_SX264 -MDLC2_SX354 -MDLC2_SX444 -MDLC2_SX84 -MDLH0_SI1960 -MDLH0_SI574 -MDLH0_SI700 -MDLH0_SX160 -MDLH0_SX250 -MDLH0_SX340 -MDLH0_SX430 -MDLH0_SX70 -MDLM0_SI1234 -MDLM0_SI1864 -MDLM0_SI604 -MDLM0_SX154 -MDLM0_SX244 -MDLM0_SX334 -MDLM0_SX424 -MDLM0_SX64 -MDLR0_SI1233 -MDLR0_SI1863 -MDLR0_SI603 -MDLR0_SX153 -MDLR0_SX243 -MDLR0_SX333 -MDLR0_SX423 -MDLR0_SX63 -MDLR1_SI1299 -MDLR1_SI1929 -MDLR1_SI669 -MDLR1_SX129 -MDLR1_SX219 -MDLR1_SX309 -MDLR1_SX39 -MDLR1_SX399 -MDMA0_SI1238 -MDMA0_SI1430 -MDMA0_SI2060 -MDMA0_SX170 -MDMA0_SX260 -MDMA0_SX350 -MDMA0_SX440 -MDMA0_SX80 -MDMT0_SI1832 -MDMT0_SI2341 -MDMT0_SI572 -MDMT0_SX122 -MDMT0_SX212 -MDMT0_SX302 -MDMT0_SX32 -MDMT0_SX392 -MDNS0_SI1011 -MDNS0_SI2271 -MDNS0_SI873 -MDNS0_SX111 -MDNS0_SX201 -MDNS0_SX21 -MDNS0_SX291 -MDNS0_SX381 -MDPB0_SI1760 -MDPB0_SI2126 -MDPB0_SI866 -MDPB0_SX146 -MDPB0_SX236 -MDPB0_SX326 -MDPB0_SX416 -MDPB0_SX56 -MDPK0_SI1053 -MDPK0_SI1683 -MDPK0_SI552 -MDPK0_SX153 -MDPK0_SX243 -MDPK0_SX333 -MDPK0_SX423 -MDPK0_SX63 -MDPS0_SI1651 -MDPS0_SI1979 -MDPS0_SI719 -MDPS0_SX179 -MDPS0_SX269 -MDPS0_SX359 -MDPS0_SX449 -MDPS0_SX89 -MDRD0_SI1382 -MDRD0_SI2012 -MDRD0_SI752 -MDRD0_SX122 -MDRD0_SX212 -MDRD0_SX302 -MDRD0_SX32 -MDRD0_SX392 -MDSJ0_SI1462 -MDSJ0_SI2092 -MDSJ0_SI832 -MDSJ0_SX112 -MDSJ0_SX22 -MDSJ0_SX292 -MDSJ0_SX382 -MDSJ0_SX438 -MDSS0_SI1881 -MDSS0_SI2087 -MDSS0_SI621 -MDSS0_SX171 -MDSS0_SX261 -MDSS0_SX351 -MDSS0_SX441 -MDSS0_SX81 -MDSS1_SI1327 -MDSS1_SI1713 -MDSS1_SI697 -MDSS1_SX157 -MDSS1_SX247 -MDSS1_SX337 -MDSS1_SX427 -MDSS1_SX67 -MDTB0_SI1200 -MDTB0_SI1830 -MDTB0_SI570 -MDTB0_SX120 -MDTB0_SX210 -MDTB0_SX300 -MDTB0_SX321 -MDTB0_SX390 -MDWD0_SI1260 -MDWD0_SI1890 -MDWD0_SI557 -MDWD0_SX180 -MDWD0_SX270 -MDWD0_SX360 -MDWD0_SX450 -MDWD0_SX90 -MDWH0_SI1168 -MDWH0_SI1925 -MDWH0_SI665 -MDWH0_SX125 -MDWH0_SX215 -MDWH0_SX305 -MDWH0_SX35 -MDWH0_SX395 -MDWM0_SI1546 -MDWM0_SI2176 -MDWM0_SI916 -MDWM0_SX106 -MDWM0_SX16 -MDWM0_SX286 -MDWM0_SX376 -MDWM0_SX433 -MEAL0_SI1547 -MEAL0_SI2177 -MEAL0_SI917 -MEAL0_SX107 -MEAL0_SX197 -MEAL0_SX287 -MEAL0_SX347 -MEAL0_SX377 -MEDR0_SI1374 -MEDR0_SI2004 -MEDR0_SI744 -MEDR0_SX114 -MEDR0_SX204 -MEDR0_SX24 -MEDR0_SX294 -MEDR0_SX384 -MEFG0_SI465 -MEFG0_SI491 -MEFG0_SI598 -MEFG0_SX105 -MEFG0_SX15 -MEFG0_SX195 -MEFG0_SX285 -MEFG0_SX375 -MEGJ0_SI1337 -MEGJ0_SI1967 -MEGJ0_SI707 -MEGJ0_SX167 -MEGJ0_SX257 -MEGJ0_SX3 -MEGJ0_SX437 -MEGJ0_SX77 -MEJL0_SI1592 -MEJL0_SI1654 -MEJL0_SI962 -MEJL0_SX152 -MEJL0_SX242 -MEJL0_SX332 -MEJL0_SX422 -MEJL0_SX62 -MEJS0_SI1240 -MEJS0_SI1870 -MEJS0_SI610 -MEJS0_SX160 -MEJS0_SX250 -MEJS0_SX340 -MEJS0_SX430 -MEJS0_SX70 -MESG0_SI1332 -MESG0_SI1962 -MESG0_SI702 -MESG0_SX162 -MESG0_SX252 -MESG0_SX342 -MESG0_SX432 -MESG0_SX72 -MESJ0_SI2039 -MESJ0_SI2257 -MESJ0_SI997 -MESJ0_SX187 -MESJ0_SX277 -MESJ0_SX367 -MESJ0_SX7 -MESJ0_SX97 -MEWM0_SI1348 -MEWM0_SI1978 -MEWM0_SI718 -MEWM0_SX178 -MEWM0_SX268 -MEWM0_SX358 -MEWM0_SX448 -MEWM0_SX88 -MFER0_SI1492 -MFER0_SI2122 -MFER0_SI862 -MFER0_SX142 -MFER0_SX232 -MFER0_SX322 -MFER0_SX412 -MFER0_SX52 -MFMC0_SI1132 -MFMC0_SI1762 -MFMC0_SI502 -MFMC0_SX142 -MFMC0_SX232 -MFMC0_SX322 -MFMC0_SX412 -MFMC0_SX52 -MFRM0_SI1155 -MFRM0_SI1717 -MFRM0_SI1785 -MFRM0_SX165 -MFRM0_SX255 -MFRM0_SX345 -MFRM0_SX435 -MFRM0_SX75 -MFWK0_SI1249 -MFWK0_SI1879 -MFWK0_SI619 -MFWK0_SX169 -MFWK0_SX259 -MFWK0_SX349 -MFWK0_SX439 -MFWK0_SX79 -MFXS0_SI1674 -MFXS0_SI2225 -MFXS0_SI2304 -MFXS0_SX144 -MFXS0_SX234 -MFXS0_SX324 -MFXS0_SX414 -MFXS0_SX54 -MFXV0_SI1005 -MFXV0_SI1342 -MFXV0_SI1635 -MFXV0_SX105 -MFXV0_SX15 -MFXV0_SX195 -MFXV0_SX285 -MFXV0_SX375 -MGAF0_SI1282 -MGAF0_SI1912 -MGAF0_SI652 -MGAF0_SX112 -MGAF0_SX202 -MGAF0_SX22 -MGAF0_SX292 -MGAF0_SX382 -MGAG0_SI1321 -MGAG0_SI645 -MGAG0_SI691 -MGAG0_SX151 -MGAG0_SX241 -MGAG0_SX331 -MGAG0_SX421 -MGAG0_SX61 -MGAK0_SI1036 -MGAK0_SI1666 -MGAK0_SI2296 -MGAK0_SX136 -MGAK0_SX226 -MGAK0_SX316 -MGAK0_SX406 -MGAK0_SX46 -MGAR0_SI1212 -MGAR0_SI1694 -MGAR0_SI1842 -MGAR0_SX132 -MGAR0_SX222 -MGAR0_SX312 -MGAR0_SX402 -MGAR0_SX42 -MGAW0_SI1165 -MGAW0_SI1802 -MGAW0_SI535 -MGAW0_SX175 -MGAW0_SX265 -MGAW0_SX355 -MGAW0_SX445 -MGAW0_SX85 -MGES0_SI1481 -MGES0_SI2111 -MGES0_SI851 -MGES0_SX131 -MGES0_SX221 -MGES0_SX311 -MGES0_SX401 -MGES0_SX41 -MGJC0_SI1256 -MGJC0_SI1335 -MGJC0_SI1965 -MGJC0_SX165 -MGJC0_SX255 -MGJC0_SX345 -MGJC0_SX435 -MGJC0_SX75 -MGRL0_SI1497 -MGRL0_SI2127 -MGRL0_SI867 -MGRL0_SX147 -MGRL0_SX237 -MGRL0_SX327 -MGRL0_SX417 -MGRL0_SX57 -MGRP0_SI1317 -MGRP0_SI1947 -MGRP0_SI687 -MGRP0_SX147 -MGRP0_SX237 -MGRP0_SX327 -MGRP0_SX417 -MGRP0_SX57 -MGSH0_SI1176 -MGSH0_SI1806 -MGSH0_SI546 -MGSH0_SX127 -MGSH0_SX186 -MGSH0_SX276 -MGSH0_SX6 -MGSH0_SX96 -MGSL0_SI1164 -MGSL0_SI534 -MGSL0_SI797 -MGSL0_SX174 -MGSL0_SX264 -MGSL0_SX354 -MGSL0_SX444 -MGSL0_SX84 -MGXP0_SI1087 -MGXP0_SI457 -MGXP0_SI525 -MGXP0_SX187 -MGXP0_SX277 -MGXP0_SX367 -MGXP0_SX7 -MGXP0_SX97 -MHBS0_SI1575 -MHBS0_SI2205 -MHBS0_SI945 -MHBS0_SX135 -MHBS0_SX225 -MHBS0_SX315 -MHBS0_SX405 -MHBS0_SX45 -MHIT0_SI1613 -MHIT0_SI2243 -MHIT0_SI983 -MHIT0_SX173 -MHIT0_SX263 -MHIT0_SX353 -MHIT0_SX443 -MHIT0_SX83 -MHJB0_SI1017 -MHJB0_SI1647 -MHJB0_SI2277 -MHJB0_SX117 -MHJB0_SX207 -MHJB0_SX27 -MHJB0_SX297 -MHJB0_SX387 -MHMG0_SI1365 -MHMG0_SI1995 -MHMG0_SI735 -MHMG0_SX105 -MHMG0_SX15 -MHMG0_SX195 -MHMG0_SX285 -MHMG0_SX375 -MHMR0_SI1119 -MHMR0_SI1692 -MHMR0_SI489 -MHMR0_SX129 -MHMR0_SX219 -MHMR0_SX309 -MHMR0_SX39 -MHMR0_SX399 -MHRM0_SI1475 -MHRM0_SI2218 -MHRM0_SI958 -MHRM0_SX148 -MHRM0_SX238 -MHRM0_SX328 -MHRM0_SX418 -MHRM0_SX58 -MHXL0_SI1772 -MHXL0_SI512 -MHXL0_SI612 -MHXL0_SX152 -MHXL0_SX242 -MHXL0_SX332 -MHXL0_SX422 -MHXL0_SX62 -MILB0_SI2163 -MILB0_SI807 -MILB0_SI903 -MILB0_SX183 -MILB0_SX273 -MILB0_SX3 -MILB0_SX363 -MILB0_SX93 -MJAC0_SI1331 -MJAC0_SI2148 -MJAC0_SI701 -MJAC0_SX251 -MJAC0_SX307 -MJAC0_SX341 -MJAC0_SX431 -MJAC0_SX71 -MJAE0_SI1524 -MJAE0_SI1999 -MJAE0_SI2154 -MJAE0_SX174 -MJAE0_SX264 -MJAE0_SX354 -MJAE0_SX444 -MJAE0_SX84 -MJAI0_SI1604 -MJAI0_SI682 -MJAI0_SI710 -MJAI0_SX164 -MJAI0_SX254 -MJAI0_SX344 -MJAI0_SX434 -MJAI0_SX74 -MJBG0_SI1232 -MJBG0_SI1724 -MJBG0_SI1862 -MJBG0_SX152 -MJBG0_SX242 -MJBG0_SX332 -MJBG0_SX422 -MJBG0_SX62 -MJDA0_SI1031 -MJDA0_SI1661 -MJDA0_SI2291 -MJDA0_SX131 -MJDA0_SX221 -MJDA0_SX311 -MJDA0_SX401 -MJDA0_SX41 -MJDC0_SI1161 -MJDC0_SI2165 -MJDC0_SI531 -MJDC0_SX171 -MJDC0_SX261 -MJDC0_SX351 -MJDC0_SX441 -MJDC0_SX81 -MJDE0_SI1120 -MJDE0_SI463 -MJDE0_SI490 -MJDE0_SX130 -MJDE0_SX220 -MJDE0_SX310 -MJDE0_SX40 -MJDE0_SX400 -MJDG0_SI1042 -MJDG0_SI1672 -MJDG0_SI1705 -MJDG0_SX142 -MJDG0_SX232 -MJDG0_SX322 -MJDG0_SX412 -MJDG0_SX52 -MJDM0_SI1340 -MJDM0_SI1937 -MJDM0_SI974 -MJDM0_SX170 -MJDM0_SX260 -MJDM0_SX350 -MJDM0_SX440 -MJDM0_SX80 -MJEB0_SI1286 -MJEB0_SI1916 -MJEB0_SI656 -MJEB0_SX170 -MJEB0_SX206 -MJEB0_SX26 -MJEB0_SX296 -MJEB0_SX386 -MJEB1_SI1467 -MJEB1_SI2097 -MJEB1_SI837 -MJEB1_SX117 -MJEB1_SX207 -MJEB1_SX27 -MJEB1_SX297 -MJEB1_SX387 -MJEE0_SI1237 -MJEE0_SI1867 -MJEE0_SI607 -MJEE0_SX157 -MJEE0_SX247 -MJEE0_SX337 -MJEE0_SX427 -MJEE0_SX67 -MJFH0_SI1107 -MJFH0_SI1737 -MJFH0_SI477 -MJFH0_SX117 -MJFH0_SX207 -MJFH0_SX27 -MJFH0_SX297 -MJFH0_SX387 -MJFR0_SI1605 -MJFR0_SI2235 -MJFR0_SI975 -MJFR0_SX165 -MJFR0_SX255 -MJFR0_SX345 -MJFR0_SX435 -MJFR0_SX75 -MJHI0_SI1328 -MJHI0_SI555 -MJHI0_SI698 -MJHI0_SX158 -MJHI0_SX248 -MJHI0_SX338 -MJHI0_SX428 -MJHI0_SX68 -MJJB0_SI1139 -MJJB0_SI1277 -MJJB0_SI1769 -MJJB0_SX149 -MJJB0_SX239 -MJJB0_SX329 -MJJB0_SX419 -MJJB0_SX59 -MJJJ0_SI1163 -MJJJ0_SI1793 -MJJJ0_SI533 -MJJJ0_SX173 -MJJJ0_SX263 -MJJJ0_SX353 -MJJJ0_SX443 -MJJJ0_SX83 -MJJM0_SI1251 -MJJM0_SI1457 -MJJM0_SI827 -MJJM0_SX107 -MJJM0_SX17 -MJJM0_SX197 -MJJM0_SX287 -MJJM0_SX377 -MJKR0_SI1201 -MJKR0_SI1831 -MJKR0_SI571 -MJKR0_SX121 -MJKR0_SX211 -MJKR0_SX301 -MJKR0_SX31 -MJKR0_SX391 -MJLB0_SI1616 -MJLB0_SI2246 -MJLB0_SI986 -MJLB0_SX176 -MJLB0_SX266 -MJLB0_SX356 -MJLB0_SX446 -MJLB0_SX86 -MJLG1_SI1012 -MJLG1_SI1642 -MJLG1_SI2272 -MJLG1_SX112 -MJLG1_SX202 -MJLG1_SX22 -MJLG1_SX292 -MJLG1_SX382 -MJLS0_SI1096 -MJLS0_SI1726 -MJLS0_SI466 -MJLS0_SX106 -MJLS0_SX16 -MJLS0_SX196 -MJLS0_SX286 -MJLS0_SX376 -MJMA0_SI1495 -MJMA0_SI2125 -MJMA0_SI865 -MJMA0_SX145 -MJMA0_SX235 -MJMA0_SX325 -MJMA0_SX415 -MJMA0_SX55 -MJMD0_SI1028 -MJMD0_SI1658 -MJMD0_SI2288 -MJMD0_SX128 -MJMD0_SX218 -MJMD0_SX308 -MJMD0_SX38 -MJMD0_SX398 -MJMM0_SI1255 -MJMM0_SI1885 -MJMM0_SI625 -MJMM0_SX175 -MJMM0_SX265 -MJMM0_SX355 -MJMM0_SX445 -MJMM0_SX85 -MJPG0_SI1191 -MJPG0_SI1821 -MJPG0_SI561 -MJPG0_SX111 -MJPG0_SX201 -MJPG0_SX21 -MJPG0_SX291 -MJPG0_SX381 -MJPM0_SI1368 -MJPM0_SI1998 -MJPM0_SI738 -MJPM0_SX108 -MJPM0_SX18 -MJPM0_SX198 -MJPM0_SX288 -MJPM0_SX378 -MJPM1_SI1897 -MJPM1_SI2280 -MJPM1_SI761 -MJPM1_SX131 -MJPM1_SX221 -MJPM1_SX311 -MJPM1_SX401 -MJPM1_SX41 -MJRA0_SI1236 -MJRA0_SI1866 -MJRA0_SI606 -MJRA0_SX156 -MJRA0_SX246 -MJRA0_SX336 -MJRA0_SX426 -MJRA0_SX66 -MJRG0_SI1366 -MJRG0_SI1996 -MJRG0_SI736 -MJRG0_SX106 -MJRG0_SX16 -MJRG0_SX286 -MJRG0_SX352 -MJRG0_SX376 -MJRH0_SI1125 -MJRH0_SI1755 -MJRH0_SI1840 -MJRH0_SX135 -MJRH0_SX225 -MJRH0_SX315 -MJRH0_SX405 -MJRH0_SX45 -MJRH1_SI1558 -MJRH1_SI1774 -MJRH1_SI514 -MJRH1_SX154 -MJRH1_SX244 -MJRH1_SX334 -MJRH1_SX424 -MJRH1_SX64 -MJRK0_SI1662 -MJRK0_SI2103 -MJRK0_SI880 -MJRK0_SX160 -MJRK0_SX250 -MJRK0_SX340 -MJRK0_SX430 -MJRK0_SX70 -MJRP0_SI1835 -MJRP0_SI1845 -MJRP0_SI585 -MJRP0_SX135 -MJRP0_SX225 -MJRP0_SX315 -MJRP0_SX405 -MJRP0_SX45 -MJSR0_SI1424 -MJSR0_SI2054 -MJSR0_SI794 -MJSR0_SX164 -MJSR0_SX254 -MJSR0_SX344 -MJSR0_SX434 -MJSR0_SX74 -MJWG0_SI2155 -MJWG0_SI813 -MJWG0_SI895 -MJWG0_SX175 -MJWG0_SX265 -MJWG0_SX355 -MJWG0_SX445 -MJWG0_SX85 -MJWS0_SI1143 -MJWS0_SI1773 -MJWS0_SI513 -MJWS0_SX153 -MJWS0_SX243 -MJWS0_SX333 -MJWS0_SX423 -MJWS0_SX63 -MJWT0_SI1291 -MJWT0_SI1381 -MJWT0_SI751 -MJWT0_SX121 -MJWT0_SX211 -MJWT0_SX301 -MJWT0_SX31 -MJWT0_SX391 -MJXA0_SI1507 -MJXA0_SI2137 -MJXA0_SI877 -MJXA0_SX157 -MJXA0_SX247 -MJXA0_SX337 -MJXA0_SX427 -MJXA0_SX67 -MJXL0_SI1172 -MJXL0_SI1795 -MJXL0_SI542 -MJXL0_SX182 -MJXL0_SX272 -MJXL0_SX362 -MJXL0_SX452 -MJXL0_SX92 -MKAG0_SI1609 -MKAG0_SI2239 -MKAG0_SI979 -MKAG0_SX169 -MKAG0_SX259 -MKAG0_SX30 -MKAG0_SX439 -MKAG0_SX79 -MKAH0_SI1528 -MKAH0_SI2158 -MKAH0_SI898 -MKAH0_SX178 -MKAH0_SX268 -MKAH0_SX358 -MKAH0_SX448 -MKAH0_SX88 -MKAJ0_SI1414 -MKAJ0_SI2044 -MKAJ0_SI784 -MKAJ0_SX154 -MKAJ0_SX244 -MKAJ0_SX334 -MKAJ0_SX424 -MKAJ0_SX64 -MKAM0_SI1250 -MKAM0_SI1316 -MKAM0_SI1465 -MKAM0_SX146 -MKAM0_SX236 -MKAM0_SX326 -MKAM0_SX416 -MKAM0_SX56 -MKDB0_SI2132 -MKDB0_SI588 -MKDB0_SI872 -MKDB0_SX152 -MKDB0_SX242 -MKDB0_SX332 -MKDB0_SX422 -MKDB0_SX62 -MKDD0_SI1567 -MKDD0_SI2197 -MKDD0_SI937 -MKDD0_SX127 -MKDD0_SX217 -MKDD0_SX307 -MKDD0_SX37 -MKDD0_SX397 -MKDT0_SI2153 -MKDT0_SI814 -MKDT0_SI893 -MKDT0_SX173 -MKDT0_SX263 -MKDT0_SX353 -MKDT0_SX443 -MKDT0_SX83 -MKES0_SI1253 -MKES0_SI1883 -MKES0_SI623 -MKES0_SX173 -MKES0_SX263 -MKES0_SX353 -MKES0_SX443 -MKES0_SX83 -MKJO0_SI1517 -MKJO0_SI2147 -MKJO0_SI887 -MKJO0_SX167 -MKJO0_SX257 -MKJO0_SX424 -MKJO0_SX437 -MKJO0_SX77 -MKLN0_SI1598 -MKLN0_SI2228 -MKLN0_SI968 -MKLN0_SX158 -MKLN0_SX248 -MKLN0_SX338 -MKLN0_SX428 -MKLN0_SX68 -MKLR0_SI1059 -MKLR0_SI1689 -MKLR0_SI2319 -MKLR0_SX159 -MKLR0_SX249 -MKLR0_SX339 -MKLR0_SX429 -MKLR0_SX69 -MKLS0_SI1437 -MKLS0_SI1533 -MKLS0_SI2067 -MKLS0_SX177 -MKLS0_SX267 -MKLS0_SX357 -MKLS0_SX447 -MKLS0_SX87 -MKLS1_SI1545 -MKLS1_SI2175 -MKLS1_SI915 -MKLS1_SX105 -MKLS1_SX15 -MKLS1_SX195 -MKLS1_SX285 -MKLS1_SX375 -MKLW0_SI1571 -MKLW0_SI1844 -MKLW0_SI2201 -MKLW0_SX131 -MKLW0_SX221 -MKLW0_SX311 -MKLW0_SX401 -MKLW0_SX41 -MKRG0_SI1491 -MKRG0_SI2121 -MKRG0_SI861 -MKRG0_SX141 -MKRG0_SX231 -MKRG0_SX31 -MKRG0_SX411 -MKRG0_SX51 -MKXL0_SI1185 -MKXL0_SI1815 -MKXL0_SI1958 -MKXL0_SX105 -MKXL0_SX15 -MKXL0_SX195 -MKXL0_SX285 -MKXL0_SX375 -MLBC0_SI1239 -MLBC0_SI1869 -MLBC0_SI609 -MLBC0_SX159 -MLBC0_SX249 -MLBC0_SX339 -MLBC0_SX429 -MLBC0_SX69 -MLEL0_SI1246 -MLEL0_SI1876 -MLEL0_SI616 -MLEL0_SX166 -MLEL0_SX256 -MLEL0_SX346 -MLEL0_SX436 -MLEL0_SX76 -MLJC0_SI1225 -MLJC0_SI1855 -MLJC0_SI595 -MLJC0_SX145 -MLJC0_SX235 -MLJC0_SX325 -MLJC0_SX415 -MLJC0_SX55 -MLJH0_SI1324 -MLJH0_SI1422 -MLJH0_SI694 -MLJH0_SX154 -MLJH0_SX244 -MLJH0_SX334 -MLJH0_SX424 -MLJH0_SX64 -MLNS0_SI1407 -MLNS0_SI2037 -MLNS0_SI777 -MLNS0_SX147 -MLNS0_SX237 -MLNS0_SX327 -MLNS0_SX417 -MLNS0_SX57 -MLSH0_SI1417 -MLSH0_SI2047 -MLSH0_SI787 -MLSH0_SX157 -MLSH0_SX247 -MLSH0_SX337 -MLSH0_SX427 -MLSH0_SX67 -MMAA0_SI1588 -MMAA0_SI2105 -MMAA0_SI845 -MMAA0_SX125 -MMAA0_SX215 -MMAA0_SX305 -MMAA0_SX35 -MMAA0_SX395 -MMAB1_SI1494 -MMAB1_SI2124 -MMAB1_SI864 -MMAB1_SX144 -MMAB1_SX234 -MMAB1_SX324 -MMAB1_SX414 -MMAB1_SX54 -MMAG0_SI1126 -MMAG0_SI1756 -MMAG0_SI496 -MMAG0_SX136 -MMAG0_SX226 -MMAG0_SX316 -MMAG0_SX406 -MMAG0_SX46 -MMAM0_SI1597 -MMAM0_SI1668 -MMAM0_SI2227 -MMAM0_SX157 -MMAM0_SX247 -MMAM0_SX337 -MMAM0_SX427 -MMAM0_SX67 -MMAR0_SI1336 -MMAR0_SI1966 -MMAR0_SI706 -MMAR0_SX166 -MMAR0_SX256 -MMAR0_SX346 -MMAR0_SX436 -MMAR0_SX76 -MMBS0_SI1151 -MMBS0_SI1781 -MMBS0_SI521 -MMBS0_SX161 -MMBS0_SX251 -MMBS0_SX341 -MMBS0_SX431 -MMBS0_SX71 -MMCC0_SI1338 -MMCC0_SI1968 -MMCC0_SI708 -MMCC0_SX168 -MMCC0_SX258 -MMCC0_SX348 -MMCC0_SX438 -MMCC0_SX78 -MMDB0_SI1358 -MMDB0_SI1617 -MMDB0_SI987 -MMDB0_SX177 -MMDB0_SX267 -MMDB0_SX357 -MMDB0_SX447 -MMDB0_SX87 -MMDG0_SI1780 -MMDG0_SI2035 -MMDG0_SI520 -MMDG0_SX160 -MMDG0_SX250 -MMDG0_SX340 -MMDG0_SX430 -MMDG0_SX70 -MMDM0_SI1311 -MMDM0_SI1941 -MMDM0_SI681 -MMDM0_SX141 -MMDM0_SX231 -MMDM0_SX321 -MMDM0_SX411 -MMDM0_SX51 -MMDM1_SI1650 -MMDM1_SI2043 -MMDM1_SI783 -MMDM1_SX153 -MMDM1_SX243 -MMDM1_SX333 -MMDM1_SX423 -MMDM1_SX63 -MMDS0_SI1343 -MMDS0_SI1973 -MMDS0_SI713 -MMDS0_SX173 -MMDS0_SX263 -MMDS0_SX353 -MMDS0_SX443 -MMDS0_SX83 -MMEA0_SI1388 -MMEA0_SI2018 -MMEA0_SI758 -MMEA0_SX128 -MMEA0_SX218 -MMEA0_SX308 -MMEA0_SX38 -MMEA0_SX398 -MMEB0_SI1357 -MMEB0_SI1987 -MMEB0_SI727 -MMEB0_SX187 -MMEB0_SX327 -MMEB0_SX367 -MMEB0_SX7 -MMEB0_SX97 -MMGC0_SI1305 -MMGC0_SI1935 -MMGC0_SI2184 -MMGC0_SX135 -MMGC0_SX225 -MMGC0_SX315 -MMGC0_SX405 -MMGC0_SX45 -MMGG0_SI1079 -MMGG0_SI1709 -MMGG0_SI2339 -MMGG0_SX179 -MMGG0_SX269 -MMGG0_SX359 -MMGG0_SX449 -MMGG0_SX89 -MMGK0_SI1322 -MMGK0_SI1952 -MMGK0_SI692 -MMGK0_SX152 -MMGK0_SX242 -MMGK0_SX332 -MMGK0_SX422 -MMGK0_SX62 -MMJB1_SI1408 -MMJB1_SI2038 -MMJB1_SI778 -MMJB1_SX148 -MMJB1_SX238 -MMJB1_SX328 -MMJB1_SX418 -MMJB1_SX58 -MMLM0_SI1527 -MMLM0_SI2150 -MMLM0_SI897 -MMLM0_SX177 -MMLM0_SX267 -MMLM0_SX357 -MMLM0_SX447 -MMLM0_SX87 -MMPM0_SI1061 -MMPM0_SI1691 -MMPM0_SI2321 -MMPM0_SX161 -MMPM0_SX251 -MMPM0_SX341 -MMPM0_SX431 -MMPM0_SX71 -MMRP0_SI2034 -MMRP0_SI717 -MMRP0_SI774 -MMRP0_SX144 -MMRP0_SX234 -MMRP0_SX324 -MMRP0_SX414 -MMRP0_SX54 -MMSM0_SI1106 -MMSM0_SI1736 -MMSM0_SI476 -MMSM0_SX116 -MMSM0_SX206 -MMSM0_SX26 -MMSM0_SX296 -MMSM0_SX386 -MMVP0_SI1284 -MMVP0_SI1914 -MMVP0_SI654 -MMVP0_SX114 -MMVP0_SX204 -MMVP0_SX294 -MMVP0_SX347 -MMVP0_SX384 -MMWB0_SI1619 -MMWB0_SI2249 -MMWB0_SI989 -MMWB0_SX179 -MMWB0_SX269 -MMWB0_SX359 -MMWB0_SX449 -MMWB0_SX89 -MMWS0_SI1518 -MMWS0_SI559 -MMWS0_SI888 -MMWS0_SX168 -MMWS0_SX258 -MMWS0_SX348 -MMWS0_SX438 -MMWS0_SX78 -MMWS1_SI1071 -MMWS1_SI1701 -MMWS1_SI2331 -MMWS1_SX261 -MMWS1_SX27 -MMWS1_SX351 -MMWS1_SX441 -MMWS1_SX81 -MMXS0_SI2136 -MMXS0_SI629 -MMXS0_SI876 -MMXS0_SX156 -MMXS0_SX246 -MMXS0_SX336 -MMXS0_SX426 -MMXS0_SX66 -MNET0_SI1446 -MNET0_SI2076 -MNET0_SI816 -MNET0_SX186 -MNET0_SX276 -MNET0_SX366 -MNET0_SX6 -MNET0_SX96 -MNTW0_SI1068 -MNTW0_SI1698 -MNTW0_SI2328 -MNTW0_SX168 -MNTW0_SX202 -MNTW0_SX258 -MNTW0_SX348 -MNTW0_SX78 -MPAR0_SI1576 -MPAR0_SI2206 -MPAR0_SI946 -MPAR0_SX136 -MPAR0_SX226 -MPAR0_SX316 -MPAR0_SX406 -MPAR0_SX46 -MPEB0_SI1034 -MPEB0_SI1860 -MPEB0_SI600 -MPEB0_SX150 -MPEB0_SX240 -MPEB0_SX330 -MPEB0_SX420 -MPEB0_SX60 -MPFU0_SI1258 -MPFU0_SI1888 -MPFU0_SI628 -MPFU0_SX178 -MPFU0_SX268 -MPFU0_SX358 -MPFU0_SX448 -MPFU0_SX88 -MPGH0_SI1554 -MPGH0_SI675 -MPGH0_SI924 -MPGH0_SX114 -MPGH0_SX204 -MPGH0_SX24 -MPGH0_SX294 -MPGH0_SX384 -MPGR0_SI1410 -MPGR0_SI2040 -MPGR0_SI780 -MPGR0_SX150 -MPGR0_SX240 -MPGR0_SX330 -MPGR0_SX420 -MPGR0_SX60 -MPGR1_SI1269 -MPGR1_SI1499 -MPGR1_SI2129 -MPGR1_SX149 -MPGR1_SX239 -MPGR1_SX329 -MPGR1_SX419 -MPGR1_SX59 -MPMB0_SI1501 -MPMB0_SI2131 -MPMB0_SI871 -MPMB0_SX151 -MPMB0_SX241 -MPMB0_SX331 -MPMB0_SX421 -MPMB0_SX61 -MPPC0_SI1412 -MPPC0_SI2042 -MPPC0_SI782 -MPPC0_SX152 -MPPC0_SX242 -MPPC0_SX332 -MPPC0_SX422 -MPPC0_SX62 -MPRB0_SI1205 -MPRB0_SI1215 -MPRB0_SI575 -MPRB0_SX125 -MPRB0_SX215 -MPRB0_SX305 -MPRB0_SX35 -MPRB0_SX395 -MPRD0_SI1431 -MPRD0_SI2061 -MPRD0_SI801 -MPRD0_SX171 -MPRD0_SX261 -MPRD0_SX351 -MPRD0_SX441 -MPRD0_SX81 -MPRK0_SI1097 -MPRK0_SI1727 -MPRK0_SI467 -MPRK0_SX107 -MPRK0_SX17 -MPRK0_SX197 -MPRK0_SX287 -MPRK0_SX377 -MPRT0_SI1210 -MPRT0_SI495 -MPRT0_SI580 -MPRT0_SX130 -MPRT0_SX220 -MPRT0_SX310 -MPRT0_SX40 -MPRT0_SX400 -MPSW0_SI1067 -MPSW0_SI1697 -MPSW0_SI2327 -MPSW0_SX167 -MPSW0_SX24 -MPSW0_SX257 -MPSW0_SX437 -MPSW0_SX77 -MRAB0_SI1224 -MRAB0_SI1854 -MRAB0_SI594 -MRAB0_SX144 -MRAB0_SX234 -MRAB0_SX324 -MRAB0_SX414 -MRAB0_SX54 -MRAB1_SI1478 -MRAB1_SI2108 -MRAB1_SI848 -MRAB1_SX128 -MRAB1_SX218 -MRAB1_SX308 -MRAB1_SX38 -MRAB1_SX398 -MRAI0_SI1954 -MRAI0_SI2052 -MRAI0_SI792 -MRAI0_SX162 -MRAI0_SX252 -MRAI0_SX342 -MRAI0_SX432 -MRAI0_SX72 -MRAM0_SI1275 -MRAM0_SI1905 -MRAM0_SI1951 -MRAM0_SX105 -MRAM0_SX15 -MRAM0_SX195 -MRAM0_SX285 -MRAM0_SX375 -MRAV0_SI1008 -MRAV0_SI1638 -MRAV0_SI2268 -MRAV0_SX108 -MRAV0_SX18 -MRAV0_SX198 -MRAV0_SX288 -MRAV0_SX378 -MRBC0_SI1665 -MRBC0_SI1859 -MRBC0_SI599 -MRBC0_SX149 -MRBC0_SX239 -MRBC0_SX329 -MRBC0_SX419 -MRBC0_SX59 -MRCG0_SI1428 -MRCG0_SI2058 -MRCG0_SI798 -MRCG0_SX168 -MRCG0_SX258 -MRCG0_SX348 -MRCG0_SX438 -MRCG0_SX78 -MRCW0_SI1371 -MRCW0_SI2001 -MRCW0_SI741 -MRCW0_SX111 -MRCW0_SX201 -MRCW0_SX21 -MRCW0_SX291 -MRCW0_SX381 -MRDD0_SI1050 -MRDD0_SI1680 -MRDD0_SI2310 -MRDD0_SX150 -MRDD0_SX240 -MRDD0_SX277 -MRDD0_SX330 -MRDD0_SX60 -MRDM0_SI1044 -MRDM0_SI1595 -MRDM0_SI965 -MRDM0_SX155 -MRDM0_SX245 -MRDM0_SX335 -MRDM0_SX425 -MRDM0_SX65 -MRDS0_SI1167 -MRDS0_SI1797 -MRDS0_SI537 -MRDS0_SX177 -MRDS0_SX267 -MRDS0_SX357 -MRDS0_SX447 -MRDS0_SX87 -MREE0_SI1104 -MREE0_SI1734 -MREE0_SI1959 -MREE0_SX114 -MREE0_SX204 -MREE0_SX24 -MREE0_SX294 -MREE0_SX384 -MREH1_SI1599 -MREH1_SI2229 -MREH1_SI969 -MREH1_SX159 -MREH1_SX249 -MREH1_SX339 -MREH1_SX429 -MREH1_SX69 -MREM0_SI1591 -MREM0_SI511 -MREM0_SI961 -MREM0_SX151 -MREM0_SX241 -MREM0_SX331 -MREM0_SX421 -MREM0_SX61 -MREW1_SI1500 -MREW1_SI2130 -MREW1_SI870 -MREW1_SX150 -MREW1_SX240 -MREW1_SX330 -MREW1_SX420 -MREW1_SX60 -MRFK0_SI1076 -MRFK0_SI1706 -MRFK0_SI2336 -MRFK0_SX176 -MRFK0_SX266 -MRFK0_SX356 -MRFK0_SX446 -MRFK0_SX86 -MRFL0_SI1156 -MRFL0_SI1786 -MRFL0_SI526 -MRFL0_SX166 -MRFL0_SX256 -MRFL0_SX346 -MRFL0_SX436 -MRFL0_SX76 -MRGM0_SI1162 -MRGM0_SI1792 -MRGM0_SI532 -MRGM0_SX172 -MRGM0_SX262 -MRGM0_SX416 -MRGM0_SX442 -MRGM0_SX82 -MRGS0_SI1356 -MRGS0_SI1986 -MRGS0_SI726 -MRGS0_SX186 -MRGS0_SX276 -MRGS0_SX366 -MRGS0_SX6 -MRGS0_SX96 -MRHL0_SI1515 -MRHL0_SI2145 -MRHL0_SI885 -MRHL0_SX165 -MRHL0_SX255 -MRHL0_SX345 -MRHL0_SX435 -MRHL0_SX75 -MRJB1_SI1020 -MRJB1_SI1413 -MRJB1_SI2021 -MRJB1_SX120 -MRJB1_SX210 -MRJB1_SX30 -MRJB1_SX300 -MRJB1_SX390 -MRJH0_SI1519 -MRJH0_SI889 -MRJH0_SI914 -MRJH0_SX169 -MRJH0_SX259 -MRJH0_SX307 -MRJH0_SX439 -MRJH0_SX79 -MRJM0_SI1095 -MRJM0_SI1228 -MRJM0_SI1858 -MRJM0_SX148 -MRJM0_SX238 -MRJM0_SX328 -MRJM0_SX418 -MRJM0_SX58 -MRJM1_SI1298 -MRJM1_SI1928 -MRJM1_SI668 -MRJM1_SX128 -MRJM1_SX218 -MRJM1_SX308 -MRJM1_SX38 -MRJM1_SX398 -MRJT0_SI1498 -MRJT0_SI1805 -MRJT0_SI868 -MRJT0_SX148 -MRJT0_SX238 -MRJT0_SX328 -MRJT0_SX418 -MRJT0_SX58 -MRKM0_SI1267 -MRKM0_SI1391 -MRKM0_SI637 -MRKM0_SX187 -MRKM0_SX277 -MRKM0_SX367 -MRKM0_SX7 -MRKM0_SX97 -MRLD0_SI1594 -MRLD0_SI2224 -MRLD0_SI964 -MRLD0_SX154 -MRLD0_SX244 -MRLD0_SX334 -MRLD0_SX424 -MRLD0_SX64 -MRLJ0_SI1420 -MRLJ0_SI2050 -MRLJ0_SI790 -MRLJ0_SX160 -MRLJ0_SX250 -MRLJ0_SX340 -MRLJ0_SX430 -MRLJ0_SX70 -MRLJ1_SI1671 -MRLJ1_SI2301 -MRLJ1_SI2332 -MRLJ1_SX141 -MRLJ1_SX231 -MRLJ1_SX321 -MRLJ1_SX411 -MRLJ1_SX51 -MRLK0_SI1468 -MRLK0_SI2140 -MRLK0_SI843 -MRLK0_SX123 -MRLK0_SX213 -MRLK0_SX303 -MRLK0_SX33 -MRLK0_SX393 -MRLR0_SI1196 -MRLR0_SI1826 -MRLR0_SI566 -MRLR0_SX116 -MRLR0_SX206 -MRLR0_SX26 -MRLR0_SX296 -MRLR0_SX386 -MRMB0_SI1581 -MRMB0_SI2211 -MRMB0_SI951 -MRMB0_SX141 -MRMB0_SX231 -MRMB0_SX321 -MRMB0_SX411 -MRMB0_SX51 -MRMG0_SI1080 -MRMG0_SI1710 -MRMG0_SI2340 -MRMG0_SX180 -MRMG0_SX270 -MRMG0_SX360 -MRMG0_SX450 -MRMG0_SX90 -MRMH0_SI1021 -MRMH0_SI1349 -MRMH0_SI2281 -MRMH0_SX121 -MRMH0_SX211 -MRMH0_SX301 -MRMH0_SX31 -MRMH0_SX391 -MRML0_SI1421 -MRML0_SI2051 -MRML0_SI791 -MRML0_SX161 -MRML0_SX251 -MRML0_SX341 -MRML0_SX431 -MRML0_SX71 -MRMS0_SI1113 -MRMS0_SI2057 -MRMS0_SI2100 -MRMS0_SX120 -MRMS0_SX210 -MRMS0_SX30 -MRMS0_SX300 -MRMS0_SX390 -MRPC1_SI1482 -MRPC1_SI2026 -MRPC1_SI2112 -MRPC1_SX132 -MRPC1_SX222 -MRPC1_SX312 -MRPC1_SX402 -MRPC1_SX42 -MRRE0_SI1334 -MRRE0_SI704 -MRRE0_SI952 -MRRE0_SX164 -MRRE0_SX254 -MRRE0_SX344 -MRRE0_SX434 -MRRE0_SX74 -MRSO0_SI1206 -MRSO0_SI1659 -MRSO0_SI2289 -MRSO0_SX129 -MRSO0_SX219 -MRSO0_SX309 -MRSO0_SX39 -MRSO0_SX399 -MRSP0_SI1429 -MRSP0_SI2059 -MRSP0_SI799 -MRSP0_SX169 -MRSP0_SX196 -MRSP0_SX259 -MRSP0_SX439 -MRSP0_SX79 -MRTC0_SI1458 -MRTC0_SI2088 -MRTC0_SI828 -MRTC0_SX108 -MRTC0_SX18 -MRTC0_SX198 -MRTC0_SX288 -MRTC0_SX378 -MRTJ0_SI1551 -MRTJ0_SI2032 -MRTJ0_SI772 -MRTJ0_SX142 -MRTJ0_SX232 -MRTJ0_SX322 -MRTJ0_SX412 -MRTJ0_SX52 -MRVG0_SI1140 -MRVG0_SI1770 -MRVG0_SI510 -MRVG0_SX150 -MRVG0_SX240 -MRVG0_SX330 -MRVG0_SX420 -MRVG0_SX60 -MRWA0_SI1603 -MRWA0_SI2233 -MRWA0_SI973 -MRWA0_SX163 -MRWA0_SX253 -MRWA0_SX343 -MRWA0_SX433 -MRWA0_SX73 -MRWS0_SI1102 -MRWS0_SI1732 -MRWS0_SI472 -MRWS0_SX112 -MRWS0_SX202 -MRWS0_SX22 -MRWS0_SX292 -MRWS0_SX382 -MRXB0_SI1585 -MRXB0_SI2215 -MRXB0_SI955 -MRXB0_SX145 -MRXB0_SX235 -MRXB0_SX325 -MRXB0_SX415 -MRXB0_SX55 -MSAH1_SI1049 -MSAH1_SI1679 -MSAH1_SI2309 -MSAH1_SX149 -MSAH1_SX239 -MSAH1_SX329 -MSAH1_SX419 -MSAH1_SX59 -MSAS0_SI1376 -MSAS0_SI2006 -MSAS0_SI746 -MSAS0_SX116 -MSAS0_SX206 -MSAS0_SX26 -MSAS0_SX296 -MSAS0_SX386 -MSAT0_SI1526 -MSAT0_SI2156 -MSAT0_SI896 -MSAT0_SX176 -MSAT0_SX266 -MSAT0_SX356 -MSAT0_SX446 -MSAT0_SX86 -MSAT1_SI1073 -MSAT1_SI1703 -MSAT1_SI2333 -MSAT1_SX173 -MSAT1_SX263 -MSAT1_SX353 -MSAT1_SX443 -MSAT1_SX83 -MSDB0_SI1007 -MSDB0_SI1637 -MSDB0_SI2267 -MSDB0_SX107 -MSDB0_SX17 -MSDB0_SX197 -MSDB0_SX287 -MSDB0_SX377 -MSDH0_SI2113 -MSDH0_SI2240 -MSDH0_SI980 -MSDH0_SX170 -MSDH0_SX260 -MSDH0_SX350 -MSDH0_SX440 -MSDH0_SX80 -MSDS0_SI1077 -MSDS0_SI1707 -MSDS0_SI2337 -MSDS0_SX177 -MSDS0_SX267 -MSDS0_SX357 -MSDS0_SX447 -MSDS0_SX87 -MSEM1_SI1440 -MSEM1_SI2070 -MSEM1_SI810 -MSEM1_SX180 -MSEM1_SX270 -MSEM1_SX360 -MSEM1_SX450 -MSEM1_SX90 -MSES0_SI1589 -MSES0_SI2216 -MSES0_SI2219 -MSES0_SX149 -MSES0_SX239 -MSES0_SX329 -MSES0_SX419 -MSES0_SX59 -MSFH0_SI1216 -MSFH0_SI1738 -MSFH0_SI586 -MSFH0_SX136 -MSFH0_SX226 -MSFH0_SX316 -MSFH0_SX406 -MSFH0_SX46 -MSFV0_SI1262 -MSFV0_SI1892 -MSFV0_SI632 -MSFV0_SX182 -MSFV0_SX272 -MSFV0_SX362 -MSFV0_SX452 -MSFV0_SX92 -MSJK0_SI1596 -MSJK0_SI2226 -MSJK0_SI966 -MSJK0_SX156 -MSJK0_SX246 -MSJK0_SX336 -MSJK0_SX426 -MSJK0_SX66 -MSMC0_SI1907 -MSMC0_SI509 -MSMC0_SI647 -MSMC0_SX107 -MSMC0_SX17 -MSMC0_SX197 -MSMC0_SX287 -MSMC0_SX377 -MSMR0_SI1150 -MSMR0_SI1405 -MSMR0_SI775 -MSMR0_SX145 -MSMR0_SX235 -MSMR0_SX325 -MSMR0_SX415 -MSMR0_SX55 -MSMS0_SI1433 -MSMS0_SI2063 -MSMS0_SI803 -MSMS0_SX173 -MSMS0_SX263 -MSMS0_SX353 -MSMS0_SX443 -MSMS0_SX83 -MSRG0_SI1221 -MSRG0_SI1851 -MSRG0_SI591 -MSRG0_SX141 -MSRG0_SX231 -MSRG0_SX321 -MSRG0_SX411 -MSRG0_SX51 -MSRR0_SI1131 -MSRR0_SI1761 -MSRR0_SI501 -MSRR0_SX141 -MSRR0_SX231 -MSRR0_SX30 -MSRR0_SX411 -MSRR0_SX51 -MSTF0_SI1396 -MSTF0_SI766 -MSTF0_SI852 -MSTF0_SX136 -MSTF0_SX226 -MSTF0_SX316 -MSTF0_SX406 -MSTF0_SX46 -MSVS0_SI1568 -MSVS0_SI2198 -MSVS0_SI938 -MSVS0_SX128 -MSVS0_SX218 -MSVS0_SX308 -MSVS0_SX38 -MSVS0_SX398 -MTAB0_SI1572 -MTAB0_SI2202 -MTAB0_SI942 -MTAB0_SX132 -MTAB0_SX222 -MTAB0_SX312 -MTAB0_SX402 -MTAB0_SX42 -MTAS0_SI1385 -MTAS0_SI2015 -MTAS0_SI755 -MTAS0_SX125 -MTAS0_SX215 -MTAS0_SX305 -MTAS0_SX35 -MTAS0_SX395 -MTAT0_SI1110 -MTAT0_SI1740 -MTAT0_SI811 -MTAT0_SX120 -MTAT0_SX210 -MTAT0_SX30 -MTAT0_SX300 -MTAT0_SX390 -MTAT1_SI1409 -MTAT1_SI1627 -MTAT1_SI779 -MTAT1_SX149 -MTAT1_SX239 -MTAT1_SX329 -MTAT1_SX419 -MTAT1_SX59 -MTBC0_SI1173 -MTBC0_SI1803 -MTBC0_SI543 -MTBC0_SX183 -MTBC0_SX273 -MTBC0_SX347 -MTBC0_SX363 -MTBC0_SX93 -MTCS0_SI1972 -MTCS0_SI2265 -MTCS0_SI712 -MTCS0_SX172 -MTCS0_SX262 -MTCS0_SX352 -MTCS0_SX442 -MTCS0_SX82 -MTDB0_SI1401 -MTDB0_SI2031 -MTDB0_SI771 -MTDB0_SX141 -MTDB0_SX231 -MTDB0_SX321 -MTDB0_SX411 -MTDB0_SX51 -MTDP0_SI1274 -MTDP0_SI1521 -MTDP0_SI2151 -MTDP0_SX171 -MTDP0_SX261 -MTDP0_SX351 -MTDP0_SX441 -MTDP0_SX81 -MTER0_SI1157 -MTER0_SI1787 -MTER0_SI527 -MTER0_SX167 -MTER0_SX17 -MTER0_SX257 -MTER0_SX437 -MTER0_SX77 -MTJG0_SI1520 -MTJG0_SI2157 -MTJG0_SI890 -MTJG0_SX170 -MTJG0_SX260 -MTJG0_SX350 -MTJG0_SX440 -MTJG0_SX80 -MTJM0_SI1226 -MTJM0_SI1856 -MTJM0_SI655 -MTJM0_SX146 -MTJM0_SX236 -MTJM0_SX326 -MTJM0_SX416 -MTJM0_SX56 -MTJS0_SI1192 -MTJS0_SI1822 -MTJS0_SI562 -MTJS0_SX112 -MTJS0_SX202 -MTJS0_SX22 -MTJS0_SX292 -MTJS0_SX382 -MTJU0_SI2020 -MTJU0_SI2269 -MTJU0_SI760 -MTJU0_SX130 -MTJU0_SX220 -MTJU0_SX310 -MTJU0_SX40 -MTJU0_SX400 -MTKD0_SI1187 -MTKD0_SI1817 -MTKD0_SI630 -MTKD0_SX107 -MTKD0_SX17 -MTKD0_SX197 -MTKD0_SX287 -MTKD0_SX377 -MTKP0_SI1023 -MTKP0_SI2283 -MTKP0_SI454 -MTKP0_SX123 -MTKP0_SX213 -MTKP0_SX303 -MTKP0_SX33 -MTKP0_SX393 -MTLB0_SI1134 -MTLB0_SI1764 -MTLB0_SI504 -MTLB0_SX144 -MTLB0_SX234 -MTLB0_SX324 -MTLB0_SX414 -MTLB0_SX54 -MTLC0_SI1313 -MTLC0_SI1477 -MTLC0_SI847 -MTLC0_SX127 -MTLC0_SX217 -MTLC0_SX307 -MTLC0_SX37 -MTLC0_SX397 -MTML0_SI1065 -MTML0_SI1695 -MTML0_SI2325 -MTML0_SX165 -MTML0_SX255 -MTML0_SX345 -MTML0_SX435 -MTML0_SX75 -MTMN0_SI1064 -MTMN0_SI2324 -MTMN0_SI582 -MTMN0_SX164 -MTMN0_SX254 -MTMN0_SX344 -MTMN0_SX434 -MTMN0_SX74 -MTMT0_SI1118 -MTMT0_SI1748 -MTMT0_SI488 -MTMT0_SX128 -MTMT0_SX218 -MTMT0_SX308 -MTMT0_SX38 -MTMT0_SX398 -MTPF0_SI1235 -MTPF0_SI1865 -MTPF0_SI605 -MTPF0_SX155 -MTPF0_SX245 -MTPF0_SX335 -MTPF0_SX425 -MTPF0_SX65 -MTPG0_SI1383 -MTPG0_SI2013 -MTPG0_SI753 -MTPG0_SX123 -MTPG0_SX213 -MTPG0_SX303 -MTPG0_SX33 -MTPG0_SX393 -MTPP0_SI1508 -MTPP0_SI2138 -MTPP0_SI878 -MTPP0_SX158 -MTPP0_SX248 -MTPP0_SX338 -MTPP0_SX428 -MTPP0_SX68 -MTPR0_SI1600 -MTPR0_SI2230 -MTPR0_SI506 -MTPR0_SX160 -MTPR0_SX250 -MTPR0_SX340 -MTPR0_SX430 -MTPR0_SX70 -MTQC0_SI1441 -MTQC0_SI2071 -MTQC0_SI480 -MTQC0_SX181 -MTQC0_SX271 -MTQC0_SX361 -MTQC0_SX451 -MTQC0_SX91 -MTRC0_SI1623 -MTRC0_SI589 -MTRC0_SI993 -MTRC0_SX170 -MTRC0_SX183 -MTRC0_SX273 -MTRC0_SX363 -MTRC0_SX93 -MTRR0_SI1548 -MTRR0_SI2178 -MTRR0_SI918 -MTRR0_SX108 -MTRR0_SX18 -MTRR0_SX198 -MTRR0_SX288 -MTRR0_SX378 -MTRT0_SI1227 -MTRT0_SI1857 -MTRT0_SI597 -MTRT0_SX147 -MTRT0_SX237 -MTRT0_SX254 -MTRT0_SX417 -MTRT0_SX57 -MTWH1_SI1512 -MTWH1_SI2142 -MTWH1_SI882 -MTWH1_SX162 -MTWH1_SX252 -MTWH1_SX342 -MTWH1_SX432 -MTWH1_SX72 -MTXS0_SI1060 -MTXS0_SI1690 -MTXS0_SI2320 -MTXS0_SX160 -MTXS0_SX250 -MTXS0_SX340 -MTXS0_SX430 -MTXS0_SX70 -MVJH0_SI1556 -MVJH0_SI2186 -MVJH0_SI926 -MVJH0_SX116 -MVJH0_SX206 -MVJH0_SX26 -MVJH0_SX296 -MVJH0_SX386 -MVLO0_SI1147 -MVLO0_SI1777 -MVLO0_SI517 -MVLO0_SX157 -MVLO0_SX247 -MVLO0_SX337 -MVLO0_SX427 -MVLO0_SX67 -MVRW0_SI1485 -MVRW0_SI2115 -MVRW0_SI855 -MVRW0_SX135 -MVRW0_SX225 -MVRW0_SX315 -MVRW0_SX405 -MVRW0_SX45 -MWAC0_SI1601 -MWAC0_SI2231 -MWAC0_SI971 -MWAC0_SX161 -MWAC0_SX251 -MWAC0_SX341 -MWAC0_SX431 -MWAC0_SX71 -MWAD0_SI1062 -MWAD0_SI1749 -MWAD0_SI2322 -MWAD0_SX162 -MWAD0_SX252 -MWAD0_SX342 -MWAD0_SX432 -MWAD0_SX72 -MWAR0_SI1045 -MWAR0_SI1675 -MWAR0_SI2305 -MWAR0_SX145 -MWAR0_SX235 -MWAR0_SX325 -MWAR0_SX415 -MWAR0_SX55 -MWCH0_SI1622 -MWCH0_SI1895 -MWCH0_SI2252 -MWCH0_SX182 -MWCH0_SX272 -MWCH0_SX362 -MWCH0_SX452 -MWCH0_SX92 -MWDK0_SI1436 -MWDK0_SI2017 -MWDK0_SI806 -MWDK0_SX176 -MWDK0_SX266 -MWDK0_SX356 -MWDK0_SX446 -MWDK0_SX86 -MWEM0_SI1320 -MWEM0_SI1393 -MWEM0_SI1950 -MWEM0_SX150 -MWEM0_SX240 -MWEM0_SX330 -MWEM0_SX420 -MWEM0_SX60 -MWGR0_SI1606 -MWGR0_SI2236 -MWGR0_SI976 -MWGR0_SX166 -MWGR0_SX256 -MWGR0_SX346 -MWGR0_SX436 -MWGR0_SX76 -MWRE0_SI1057 -MWRE0_SI1687 -MWRE0_SI2317 -MWRE0_SX157 -MWRE0_SX247 -MWRE0_SX337 -MWRE0_SX427 -MWRE0_SX67 -MWRP0_SI1443 -MWRP0_SI1525 -MWRP0_SI2073 -MWRP0_SX183 -MWRP0_SX273 -MWRP0_SX3 -MWRP0_SX363 -MWRP0_SX93 -MWSB0_SI1626 -MWSB0_SI2256 -MWSB0_SI996 -MWSB0_SX186 -MWSB0_SX276 -MWSB0_SX366 -MWSB0_SX6 -MWSB0_SX96 -MWSH0_SI1426 -MWSH0_SI2266 -MWSH0_SI796 -MWSH0_SX166 -MWSH0_SX256 -MWSH0_SX346 -MWSH0_SX436 -MWSH0_SX76 -MZMB0_SI1166 -MZMB0_SI1796 -MZMB0_SI536 -MZMB0_SX176 -MZMB0_SX266 -MZMB0_SX356 -MZMB0_SX446 -MZMB0_SX86 diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/config/timit_matched/train_text.uid b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/config/timit_matched/train_text.uid deleted file mode 100644 index c39fd0b91d51e0ae15caf1e9701d0d9ef51ee21b..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/config/timit_matched/train_text.uid +++ /dev/null @@ -1,3696 +0,0 @@ -FAEM0_SI1392 -FAEM0_SI2022 -FAEM0_SI762 -FAEM0_SX132 -FAEM0_SX222 -FAEM0_SX312 -FAEM0_SX402 -FAEM0_SX42 -FAJW0_SI1263 -FAJW0_SI1893 -FAJW0_SI633 -FAJW0_SX183 -FAJW0_SX273 -FAJW0_SX3 -FAJW0_SX363 -FAJW0_SX93 -FALK0_SI1086 -FALK0_SI456 -FALK0_SI658 -FALK0_SX186 -FALK0_SX276 -FALK0_SX366 -FALK0_SX6 -FALK0_SX96 -FALR0_SI1325 -FALR0_SI1955 -FALR0_SI695 -FALR0_SX155 -FALR0_SX245 -FALR0_SX335 -FALR0_SX425 -FALR0_SX65 -FAPB0_SI1063 -FAPB0_SI1693 -FAPB0_SI2323 -FAPB0_SX163 -FAPB0_SX253 -FAPB0_SX343 -FAPB0_SX433 -FAPB0_SX73 -FBAS0_SI1387 -FBAS0_SI1472 -FBAS0_SI2066 -FBAS0_SX127 -FBAS0_SX217 -FBAS0_SX307 -FBAS0_SX37 -FBAS0_SX397 -FBCG1_SI1612 -FBCG1_SI2242 -FBCG1_SI982 -FBCG1_SX172 -FBCG1_SX262 -FBCG1_SX352 -FBCG1_SX442 -FBCG1_SX82 -FBCH0_SI1586 -FBCH0_SI956 -FBCH0_SI959 -FBCH0_SX146 -FBCH0_SX236 -FBCH0_SX326 -FBCH0_SX416 -FBCH0_SX56 -FBJL0_SI1552 -FBJL0_SI2182 -FBJL0_SI922 -FBJL0_SX112 -FBJL0_SX202 -FBJL0_SX22 -FBJL0_SX292 -FBJL0_SX382 -FBLV0_SI1058 -FBLV0_SI1688 -FBLV0_SI2318 -FBLV0_SX158 -FBLV0_SX248 -FBLV0_SX338 -FBLV0_SX428 -FBLV0_SX68 -FBMH0_SI1136 -FBMH0_SI1766 -FBMH0_SI970 -FBMH0_SX146 -FBMH0_SX236 -FBMH0_SX326 -FBMH0_SX416 -FBMH0_SX56 -FBMJ0_SI1776 -FBMJ0_SI516 -FBMJ0_SI815 -FBMJ0_SX156 -FBMJ0_SX246 -FBMJ0_SX336 -FBMJ0_SX426 -FBMJ0_SX66 -FCAG0_SI1503 -FCAG0_SI1641 -FCAG0_SI2133 -FCAG0_SX153 -FCAG0_SX243 -FCAG0_SX333 -FCAG0_SX423 -FCAG0_SX63 -FCAJ0_SI1479 -FCAJ0_SI1804 -FCAJ0_SI849 -FCAJ0_SX129 -FCAJ0_SX219 -FCAJ0_SX309 -FCAJ0_SX39 -FCAJ0_SX399 -FCDR1_SI1186 -FCDR1_SI1816 -FCDR1_SI556 -FCDR1_SX106 -FCDR1_SX16 -FCDR1_SX196 -FCDR1_SX286 -FCDR1_SX376 -FCEG0_SI1248 -FCEG0_SI1878 -FCEG0_SI618 -FCEG0_SX168 -FCEG0_SX258 -FCEG0_SX348 -FCEG0_SX438 -FCEG0_SX78 -FCJF0_SI1027 -FCJF0_SI1657 -FCJF0_SI648 -FCJF0_SX127 -FCJF0_SX217 -FCJF0_SX307 -FCJF0_SX37 -FCJF0_SX397 -FCJS0_SI1607 -FCJS0_SI2237 -FCJS0_SI977 -FCJS0_SX167 -FCJS0_SX257 -FCJS0_SX347 -FCJS0_SX437 -FCJS0_SX77 -FCKE0_SI1111 -FCKE0_SI1741 -FCKE0_SI481 -FCKE0_SX121 -FCKE0_SX211 -FCKE0_SX301 -FCKE0_SX31 -FCKE0_SX391 -FCLT0_SI1438 -FCLT0_SI2068 -FCLT0_SI808 -FCLT0_SX178 -FCLT0_SX268 -FCLT0_SX358 -FCLT0_SX448 -FCLT0_SX88 -FCMG0_SI1142 -FCMG0_SI1242 -FCMG0_SI1872 -FCMG0_SX162 -FCMG0_SX252 -FCMG0_SX342 -FCMG0_SX432 -FCMG0_SX72 -FCMM0_SI1083 -FCMM0_SI1957 -FCMM0_SI453 -FCMM0_SX183 -FCMM0_SX273 -FCMM0_SX363 -FCMM0_SX420 -FCMM0_SX93 -FCRZ0_SI1913 -FCRZ0_SI2053 -FCRZ0_SI793 -FCRZ0_SX163 -FCRZ0_SX253 -FCRZ0_SX343 -FCRZ0_SX433 -FCRZ0_SX73 -FCYL0_SI1297 -FCYL0_SI1927 -FCYL0_SI667 -FCYL0_SX127 -FCYL0_SX217 -FCYL0_SX349 -FCYL0_SX37 -FCYL0_SX397 -FDAS1_SI1461 -FDAS1_SI2091 -FDAS1_SI831 -FDAS1_SX111 -FDAS1_SX201 -FDAS1_SX21 -FDAS1_SX291 -FDAS1_SX381 -FDAW0_SI1271 -FDAW0_SI1406 -FDAW0_SI2036 -FDAW0_SX146 -FDAW0_SX236 -FDAW0_SX326 -FDAW0_SX416 -FDAW0_SX56 -FDFB0_SI1318 -FDFB0_SI1948 -FDFB0_SI2010 -FDFB0_SX148 -FDFB0_SX238 -FDFB0_SX328 -FDFB0_SX418 -FDFB0_SX58 -FDJH0_SI1565 -FDJH0_SI2195 -FDJH0_SI935 -FDJH0_SX125 -FDJH0_SX215 -FDJH0_SX305 -FDJH0_SX35 -FDJH0_SX395 -FDKN0_SI1081 -FDKN0_SI1202 -FDKN0_SI1711 -FDKN0_SX181 -FDKN0_SX271 -FDKN0_SX361 -FDKN0_SX451 -FDKN0_SX91 -FDML0_SI1149 -FDML0_SI1779 -FDML0_SI2075 -FDML0_SX159 -FDML0_SX249 -FDML0_SX339 -FDML0_SX429 -FDML0_SX69 -FDMY0_SI1197 -FDMY0_SI567 -FDMY0_SI714 -FDMY0_SX117 -FDMY0_SX207 -FDMY0_SX27 -FDMY0_SX297 -FDMY0_SX387 -FDNC0_SI1278 -FDNC0_SI1908 -FDNC0_SI2287 -FDNC0_SX108 -FDNC0_SX18 -FDNC0_SX198 -FDNC0_SX288 -FDNC0_SX378 -FDTD0_SI1561 -FDTD0_SI2191 -FDTD0_SI931 -FDTD0_SX121 -FDTD0_SX211 -FDTD0_SX301 -FDTD0_SX321 -FDTD0_SX391 -FDXW0_SI1511 -FDXW0_SI2141 -FDXW0_SI881 -FDXW0_SX161 -FDXW0_SX251 -FDXW0_SX341 -FDXW0_SX431 -FDXW0_SX71 -FEAC0_SI1245 -FEAC0_SI1875 -FEAC0_SI615 -FEAC0_SX165 -FEAC0_SX255 -FEAC0_SX345 -FEAC0_SX435 -FEAC0_SX75 -FEAR0_SI1252 -FEAR0_SI1882 -FEAR0_SI622 -FEAR0_SX172 -FEAR0_SX262 -FEAR0_SX352 -FEAR0_SX442 -FEAR0_SX82 -FECD0_SI1418 -FECD0_SI2048 -FECD0_SI788 -FECD0_SX158 -FECD0_SX248 -FECD0_SX338 -FECD0_SX428 -FECD0_SX68 -FEEH0_SI1112 -FEEH0_SI1742 -FEEH0_SI471 -FEEH0_SX122 -FEEH0_SX212 -FEEH0_SX302 -FEEH0_SX32 -FEEH0_SX392 -FEME0_SI1505 -FEME0_SI2135 -FEME0_SI875 -FEME0_SX155 -FEME0_SX245 -FEME0_SX335 -FEME0_SX425 -FEME0_SX65 -FETB0_SI1148 -FETB0_SI1778 -FETB0_SI518 -FETB0_SX158 -FETB0_SX248 -FETB0_SX338 -FETB0_SX428 -FETB0_SX68 -FEXM0_SI1101 -FEXM0_SI1731 -FEXM0_SI482 -FEXM0_SX111 -FEXM0_SX201 -FEXM0_SX291 -FEXM0_SX366 -FEXM0_SX381 -FGCS0_SI1486 -FGCS0_SI2116 -FGCS0_SI856 -FGCS0_SX136 -FGCS0_SX226 -FGCS0_SX316 -FGCS0_SX406 -FGCS0_SX46 -FGDP0_SI1618 -FGDP0_SI2248 -FGDP0_SI988 -FGDP0_SX178 -FGDP0_SX268 -FGDP0_SX358 -FGDP0_SX448 -FGDP0_SX88 -FGMB0_SI1145 -FGMB0_SI1775 -FGMB0_SI515 -FGMB0_SX155 -FGMB0_SX245 -FGMB0_SX335 -FGMB0_SX425 -FGMB0_SX65 -FGRW0_SI1152 -FGRW0_SI1782 -FGRW0_SI1990 -FGRW0_SX162 -FGRW0_SX252 -FGRW0_SX342 -FGRW0_SX432 -FGRW0_SX72 -FHLM0_SI1560 -FHLM0_SI2190 -FHLM0_SI930 -FHLM0_SX120 -FHLM0_SX210 -FHLM0_SX300 -FHLM0_SX349 -FHLM0_SX390 -FHXS0_SI1075 -FHXS0_SI2302 -FHXS0_SI2335 -FHXS0_SX175 -FHXS0_SX265 -FHXS0_SX355 -FHXS0_SX445 -FHXS0_SX85 -FJDM2_SI1582 -FJDM2_SI1964 -FJDM2_SI2212 -FJDM2_SX142 -FJDM2_SX232 -FJDM2_SX322 -FJDM2_SX412 -FJDM2_SX52 -FJEN0_SI1047 -FJEN0_SI1677 -FJEN0_SI2307 -FJEN0_SX147 -FJEN0_SX237 -FJEN0_SX327 -FJEN0_SX417 -FJEN0_SX57 -FJHK0_SI1022 -FJHK0_SI1652 -FJHK0_SI2282 -FJHK0_SX122 -FJHK0_SX212 -FJHK0_SX302 -FJHK0_SX32 -FJHK0_SX392 -FJKL0_SI1562 -FJKL0_SI2192 -FJKL0_SI932 -FJKL0_SX122 -FJKL0_SX212 -FJKL0_SX302 -FJKL0_SX32 -FJKL0_SX392 -FJLG0_SI1506 -FJLG0_SI1889 -FJLG0_SI2306 -FJLG0_SX179 -FJLG0_SX269 -FJLG0_SX359 -FJLG0_SX449 -FJLG0_SX89 -FJLR0_SI1231 -FJLR0_SI1861 -FJLR0_SI601 -FJLR0_SX151 -FJLR0_SX241 -FJLR0_SX331 -FJLR0_SX421 -FJLR0_SX61 -FJRB0_SI1302 -FJRB0_SI1932 -FJRB0_SI672 -FJRB0_SX132 -FJRB0_SX222 -FJRB0_SX312 -FJRB0_SX402 -FJRB0_SX42 -FJRP1_SI1432 -FJRP1_SI2062 -FJRP1_SI802 -FJRP1_SX172 -FJRP1_SX262 -FJRP1_SX352 -FJRP1_SX442 -FJRP1_SX82 -FJSK0_SI1052 -FJSK0_SI1682 -FJSK0_SI2312 -FJSK0_SX152 -FJSK0_SX242 -FJSK0_SX332 -FJSK0_SX422 -FJSK0_SX62 -FJSP0_SI1434 -FJSP0_SI1763 -FJSP0_SI804 -FJSP0_SX174 -FJSP0_SX264 -FJSP0_SX354 -FJSP0_SX444 -FJSP0_SX84 -FJWB1_SI2055 -FJWB1_SI748 -FJWB1_SI795 -FJWB1_SX165 -FJWB1_SX255 -FJWB1_SX345 -FJWB1_SX435 -FJWB1_SX75 -FJXM0_SI1211 -FJXM0_SI1971 -FJXM0_SI581 -FJXM0_SX131 -FJXM0_SX221 -FJXM0_SX311 -FJXM0_SX401 -FJXM0_SX41 -FJXP0_SI1122 -FJXP0_SI1752 -FJXP0_SI492 -FJXP0_SX132 -FJXP0_SX222 -FJXP0_SX312 -FJXP0_SX402 -FJXP0_SX42 -FKAA0_SI1208 -FKAA0_SI1838 -FKAA0_SI578 -FKAA0_SX128 -FKAA0_SX218 -FKAA0_SX308 -FKAA0_SX38 -FKAA0_SX398 -FKDE0_SI1141 -FKDE0_SI1771 -FKDE0_SI2221 -FKDE0_SX151 -FKDE0_SX241 -FKDE0_SX331 -FKDE0_SX421 -FKDE0_SX61 -FKDW0_SI1207 -FKDW0_SI1891 -FKDW0_SI577 -FKDW0_SX127 -FKDW0_SX217 -FKDW0_SX307 -FKDW0_SX37 -FKDW0_SX397 -FKFB0_SI1608 -FKFB0_SI2238 -FKFB0_SI978 -FKFB0_SX168 -FKFB0_SX258 -FKFB0_SX348 -FKFB0_SX438 -FKFB0_SX78 -FKKH0_SI1290 -FKKH0_SI1920 -FKKH0_SI660 -FKKH0_SX120 -FKKH0_SX210 -FKKH0_SX30 -FKKH0_SX300 -FKKH0_SX390 -FKLC0_SI1615 -FKLC0_SI2245 -FKLC0_SI985 -FKLC0_SX175 -FKLC0_SX265 -FKLC0_SX355 -FKLC0_SX445 -FKLC0_SX85 -FKLC1_SI1048 -FKLC1_SI1678 -FKLC1_SI2308 -FKLC1_SX148 -FKLC1_SX238 -FKLC1_SX328 -FKLC1_SX418 -FKLC1_SX58 -FKLH0_SI1257 -FKLH0_SI1887 -FKLH0_SI627 -FKLH0_SX177 -FKLH0_SX267 -FKLH0_SX357 -FKLH0_SX447 -FKLH0_SX87 -FKSR0_SI1117 -FKSR0_SI1747 -FKSR0_SI487 -FKSR0_SX161 -FKSR0_SX217 -FKSR0_SX366 -FKSR0_SX37 -FKSR0_SX397 -FLAC0_SI1339 -FLAC0_SI2161 -FLAC0_SI901 -FLAC0_SX181 -FLAC0_SX271 -FLAC0_SX361 -FLAC0_SX451 -FLAC0_SX91 -FLAG0_SI1464 -FLAG0_SI2094 -FLAG0_SI834 -FLAG0_SX114 -FLAG0_SX204 -FLAG0_SX24 -FLAG0_SX294 -FLAG0_SX384 -FLEH0_SI1051 -FLEH0_SI1681 -FLEH0_SI2311 -FLEH0_SX151 -FLEH0_SX241 -FLEH0_SX331 -FLEH0_SX421 -FLEH0_SX61 -FLET0_SI1137 -FLET0_SI1767 -FLET0_SI507 -FLET0_SX147 -FLET0_SX237 -FLET0_SX277 -FLET0_SX417 -FLET0_SX57 -FLHD0_SI1344 -FLHD0_SI1827 -FLHD0_SI1974 -FLHD0_SX174 -FLHD0_SX264 -FLHD0_SX354 -FLHD0_SX444 -FLHD0_SX84 -FLJA0_SI1078 -FLJA0_SI1708 -FLJA0_SI2338 -FLJA0_SX178 -FLJA0_SX268 -FLJA0_SX358 -FLJA0_SX448 -FLJA0_SX88 -FLJD0_SI1516 -FLJD0_SI2146 -FLJD0_SI886 -FLJD0_SX166 -FLJD0_SX256 -FLJD0_SX346 -FLJD0_SX436 -FLJD0_SX76 -FLJG0_SI1611 -FLJG0_SI2241 -FLJG0_SI981 -FLJG0_SX171 -FLJG0_SX261 -FLJG0_SX351 -FLJG0_SX441 -FLJG0_SX81 -FLKM0_SI1880 -FLKM0_SI620 -FLKM0_SI686 -FLKM0_SX116 -FLKM0_SX260 -FLKM0_SX350 -FLKM0_SX440 -FLKM0_SX80 -FLMA0_SI1243 -FLMA0_SI1873 -FLMA0_SI613 -FLMA0_SX163 -FLMA0_SX253 -FLMA0_SX343 -FLMA0_SX433 -FLMA0_SX73 -FLMC0_SI1372 -FLMC0_SI2002 -FLMC0_SI742 -FLMC0_SX112 -FLMC0_SX22 -FLMC0_SX292 -FLMC0_SX336 -FLMC0_SX382 -FLMK0_SI1035 -FLMK0_SI1229 -FLMK0_SI2295 -FLMK0_SX135 -FLMK0_SX225 -FLMK0_SX315 -FLMK0_SX405 -FLMK0_SX45 -FLOD0_SI1287 -FLOD0_SI1917 -FLOD0_SI657 -FLOD0_SX117 -FLOD0_SX171 -FLOD0_SX207 -FLOD0_SX297 -FLOD0_SX387 -FLTM0_SI1070 -FLTM0_SI1700 -FLTM0_SI2330 -FLTM0_SX170 -FLTM0_SX260 -FLTM0_SX350 -FLTM0_SX440 -FLTM0_SX80 -FMAH1_SI1509 -FMAH1_SI2139 -FMAH1_SI879 -FMAH1_SX159 -FMAH1_SX249 -FMAH1_SX339 -FMAH1_SX429 -FMAH1_SX69 -FMBG0_SI1160 -FMBG0_SI1790 -FMBG0_SI2264 -FMBG0_SX260 -FMBG0_SX3 -FMBG0_SX350 -FMBG0_SX440 -FMBG0_SX80 -FMEM0_SI1377 -FMEM0_SI2007 -FMEM0_SI747 -FMEM0_SX117 -FMEM0_SX207 -FMEM0_SX297 -FMEM0_SX333 -FMEM0_SX387 -FMJB0_SI1177 -FMJB0_SI1807 -FMJB0_SI547 -FMJB0_SX187 -FMJB0_SX277 -FMJB0_SX367 -FMJB0_SX7 -FMJB0_SX97 -FMJF0_SI1254 -FMJF0_SI1884 -FMJF0_SI624 -FMJF0_SX174 -FMJF0_SX264 -FMJF0_SX354 -FMJF0_SX444 -FMJF0_SX84 -FMJU0_SI1389 -FMJU0_SI2019 -FMJU0_SI759 -FMJU0_SX129 -FMJU0_SX219 -FMJU0_SX309 -FMJU0_SX39 -FMJU0_SX399 -FMKC0_SI1041 -FMKC0_SI1072 -FMKC0_SI1702 -FMKC0_SX172 -FMKC0_SX262 -FMKC0_SX352 -FMKC0_SX442 -FMKC0_SX82 -FMKF0_SI1018 -FMKF0_SI1536 -FMKF0_SI906 -FMKF0_SX186 -FMKF0_SX276 -FMKF0_SX366 -FMKF0_SX6 -FMKF0_SX96 -FMMH0_SI1537 -FMMH0_SI2167 -FMMH0_SI907 -FMMH0_SX187 -FMMH0_SX367 -FMMH0_SX420 -FMMH0_SX7 -FMMH0_SX97 -FMPG0_SI1602 -FMPG0_SI2232 -FMPG0_SI972 -FMPG0_SX162 -FMPG0_SX252 -FMPG0_SX342 -FMPG0_SX432 -FMPG0_SX72 -FNKL0_SI1522 -FNKL0_SI2152 -FNKL0_SI892 -FNKL0_SX172 -FNKL0_SX196 -FNKL0_SX262 -FNKL0_SX442 -FNKL0_SX82 -FNTB0_SI1203 -FNTB0_SI573 -FNTB0_SI679 -FNTB0_SX123 -FNTB0_SX213 -FNTB0_SX303 -FNTB0_SX33 -FNTB0_SX393 -FPAB1_SI1471 -FPAB1_SI2101 -FPAB1_SI841 -FPAB1_SX121 -FPAB1_SX211 -FPAB1_SX301 -FPAB1_SX31 -FPAB1_SX391 -FPAC0_SI1921 -FPAC0_SI2011 -FPAC0_SI661 -FPAC0_SX121 -FPAC0_SX211 -FPAC0_SX301 -FPAC0_SX31 -FPAC0_SX391 -FPAD0_SI1346 -FPAD0_SI1976 -FPAD0_SI716 -FPAD0_SX176 -FPAD0_SX266 -FPAD0_SX356 -FPAD0_SX446 -FPAD0_SX86 -FPAF0_SI1054 -FPAF0_SI1684 -FPAF0_SI2314 -FPAF0_SX154 -FPAF0_SX244 -FPAF0_SX334 -FPAF0_SX424 -FPAF0_SX64 -FPAZ0_SI1593 -FPAZ0_SI2223 -FPAZ0_SI963 -FPAZ0_SX153 -FPAZ0_SX243 -FPAZ0_SX27 -FPAZ0_SX423 -FPAZ0_SX63 -FPJF0_SI1046 -FPJF0_SI1259 -FPJF0_SI1676 -FPJF0_SX146 -FPJF0_SX236 -FPJF0_SX326 -FPJF0_SX352 -FPJF0_SX56 -FPLS0_SI1590 -FPLS0_SI2220 -FPLS0_SI960 -FPLS0_SX150 -FPLS0_SX240 -FPLS0_SX3 -FPLS0_SX330 -FPLS0_SX60 -FPMY0_SI1153 -FPMY0_SI1783 -FPMY0_SI523 -FPMY0_SX163 -FPMY0_SX196 -FPMY0_SX253 -FPMY0_SX343 -FPMY0_SX73 -FREH0_SI1315 -FREH0_SI1945 -FREH0_SI685 -FREH0_SX145 -FREH0_SX235 -FREH0_SX325 -FREH0_SX415 -FREH0_SX55 -FRJB0_SI1427 -FRJB0_SI1470 -FRJB0_SI1794 -FRJB0_SX167 -FRJB0_SX257 -FRJB0_SX347 -FRJB0_SX437 -FRJB0_SX77 -FRLL0_SI1514 -FRLL0_SI805 -FRLL0_SI884 -FRLL0_SX164 -FRLL0_SX254 -FRLL0_SX344 -FRLL0_SX434 -FRLL0_SX74 -FSAG0_SI1323 -FSAG0_SI1953 -FSAG0_SI693 -FSAG0_SX153 -FSAG0_SX243 -FSAG0_SX333 -FSAG0_SX423 -FSAG0_SX63 -FSAH0_SI1244 -FSAH0_SI1874 -FSAH0_SI614 -FSAH0_SX164 -FSAH0_SX327 -FSAH0_SX344 -FSAH0_SX434 -FSAH0_SX74 -FSAK0_SI1300 -FSAK0_SI1930 -FSAK0_SI670 -FSAK0_SX130 -FSAK0_SX220 -FSAK0_SX310 -FSAK0_SX40 -FSAK0_SX400 -FSBK0_SI1069 -FSBK0_SI1699 -FSBK0_SI2329 -FSBK0_SX169 -FSBK0_SX259 -FSBK0_SX349 -FSBK0_SX439 -FSBK0_SX79 -FSCN0_SI1886 -FSCN0_SI626 -FSCN0_SI705 -FSCN0_SX176 -FSCN0_SX266 -FSCN0_SX356 -FSCN0_SX446 -FSCN0_SX86 -FSDC0_SI1312 -FSDC0_SI1942 -FSDC0_SI2234 -FSDC0_SX142 -FSDC0_SX232 -FSDC0_SX322 -FSDC0_SX412 -FSDC0_SX52 -FSDJ0_SI1115 -FSDJ0_SI1745 -FSDJ0_SI485 -FSDJ0_SX125 -FSDJ0_SX215 -FSDJ0_SX305 -FSDJ0_SX35 -FSDJ0_SX395 -FSGF0_SI1557 -FSGF0_SI2187 -FSGF0_SI927 -FSGF0_SX117 -FSGF0_SX207 -FSGF0_SX27 -FSGF0_SX297 -FSGF0_SX387 -FSJG0_SI1570 -FSJG0_SI2200 -FSJG0_SI940 -FSJG0_SX130 -FSJG0_SX220 -FSJG0_SX310 -FSJG0_SX40 -FSJG0_SX400 -FSJK1_SI1025 -FSJK1_SI2285 -FSJK1_SI696 -FSJK1_SX125 -FSJK1_SX215 -FSJK1_SX305 -FSJK1_SX35 -FSJK1_SX395 -FSJS0_SI1171 -FSJS0_SI1801 -FSJS0_SI541 -FSJS0_SX181 -FSJS0_SX271 -FSJS0_SX361 -FSJS0_SX451 -FSJS0_SX91 -FSJW0_SI1333 -FSJW0_SI1963 -FSJW0_SI703 -FSJW0_SX163 -FSJW0_SX253 -FSJW0_SX343 -FSJW0_SX433 -FSJW0_SX73 -FSKC0_SI1416 -FSKC0_SI2046 -FSKC0_SI786 -FSKC0_SX156 -FSKC0_SX246 -FSKC0_SX336 -FSKC0_SX426 -FSKC0_SX66 -FSKL0_SI1529 -FSKL0_SI2159 -FSKL0_SI899 -FSKL0_SX179 -FSKL0_SX269 -FSKL0_SX359 -FSKL0_SX449 -FSKL0_SX89 -FSKP0_SI1098 -FSKP0_SI1728 -FSKP0_SI468 -FSKP0_SX108 -FSKP0_SX18 -FSKP0_SX198 -FSKP0_SX288 -FSKP0_SX378 -FSLS0_SI1056 -FSLS0_SI1686 -FSLS0_SI2316 -FSLS0_SX156 -FSLS0_SX202 -FSLS0_SX246 -FSLS0_SX426 -FSLS0_SX66 -FSMA0_SI1621 -FSMA0_SI2251 -FSMA0_SI991 -FSMA0_SX181 -FSMA0_SX271 -FSMA0_SX361 -FSMA0_SX451 -FSMA0_SX91 -FSMM0_SI1314 -FSMM0_SI1944 -FSMM0_SI684 -FSMM0_SX144 -FSMM0_SX234 -FSMM0_SX324 -FSMM0_SX414 -FSMM0_SX54 -FSMS1_SI1504 -FSMS1_SI2134 -FSMS1_SI874 -FSMS1_SX154 -FSMS1_SX244 -FSMS1_SX334 -FSMS1_SX347 -FSMS1_SX64 -FSPM0_SI1241 -FSPM0_SI1871 -FSPM0_SI611 -FSPM0_SX161 -FSPM0_SX251 -FSPM0_SX341 -FSPM0_SX431 -FSPM0_SX71 -FSRH0_SI1719 -FSRH0_SI1931 -FSRH0_SI671 -FSRH0_SX131 -FSRH0_SX221 -FSRH0_SX311 -FSRH0_SX401 -FSRH0_SX41 -FSSB0_SI1082 -FSSB0_SI1712 -FSSB0_SI2342 -FSSB0_SX182 -FSSB0_SX272 -FSSB0_SX362 -FSSB0_SX452 -FSSB0_SX92 -FTAJ0_SI1329 -FTAJ0_SI474 -FTAJ0_SI699 -FTAJ0_SX159 -FTAJ0_SX249 -FTAJ0_SX339 -FTAJ0_SX429 -FTAJ0_SX69 -FTBR0_SI1402 -FTBR0_SI2181 -FTBR0_SI921 -FTBR0_SX111 -FTBR0_SX201 -FTBR0_SX21 -FTBR0_SX291 -FTBR0_SX381 -FTBW0_SI1345 -FTBW0_SI1975 -FTBW0_SI715 -FTBW0_SX175 -FTBW0_SX265 -FTBW0_SX355 -FTBW0_SX445 -FTBW0_SX85 -FTLG0_SI1743 -FTLG0_SI483 -FTLG0_SI840 -FTLG0_SX123 -FTLG0_SX213 -FTLG0_SX303 -FTLG0_SX33 -FTLG0_SX393 -FTMG0_SI1532 -FTMG0_SI2162 -FTMG0_SI902 -FTMG0_SX182 -FTMG0_SX272 -FTMG0_SX362 -FTMG0_SX452 -FTMG0_SX92 -FVFB0_SI1032 -FVFB0_SI1510 -FVFB0_SI2292 -FVFB0_SX132 -FVFB0_SX222 -FVFB0_SX312 -FVFB0_SX402 -FVFB0_SX42 -FVKB0_SI1159 -FVKB0_SI1789 -FVKB0_SI529 -FVKB0_SX169 -FVKB0_SX259 -FVKB0_SX349 -FVKB0_SX439 -FVKB0_SX79 -FVMH0_SI1466 -FVMH0_SI2096 -FVMH0_SI836 -FVMH0_SX116 -FVMH0_SX206 -FVMH0_SX26 -FVMH0_SX296 -FVMH0_SX386 -MABC0_SI1620 -MABC0_SI2041 -MABC0_SI781 -MABC0_SX151 -MABC0_SX241 -MABC0_SX331 -MABC0_SX421 -MABC0_SX61 -MADC0_SI1367 -MADC0_SI1997 -MADC0_SI737 -MADC0_SX107 -MADC0_SX17 -MADC0_SX197 -MADC0_SX287 -MADC0_SX377 -MADD0_SI1295 -MADD0_SI1798 -MADD0_SI538 -MADD0_SX178 -MADD0_SX268 -MADD0_SX358 -MADD0_SX448 -MADD0_SX88 -MAEB0_SI1411 -MAEB0_SI2250 -MAEB0_SI990 -MAEB0_SX180 -MAEB0_SX270 -MAEB0_SX360 -MAEB0_SX450 -MAEB0_SX90 -MAEO0_SI1326 -MAEO0_SI1655 -MAEO0_SI1956 -MAEO0_SX156 -MAEO0_SX246 -MAEO0_SX336 -MAEO0_SX426 -MAEO0_SX66 -MAFM0_SI1569 -MAFM0_SI2199 -MAFM0_SI939 -MAFM0_SX129 -MAFM0_SX219 -MAFM0_SX309 -MAFM0_SX39 -MAFM0_SX399 -MAJP0_SI1074 -MAJP0_SI1704 -MAJP0_SI2334 -MAJP0_SX174 -MAJP0_SX264 -MAJP0_SX354 -MAJP0_SX444 -MAJP0_SX84 -MAKB0_SI1016 -MAKB0_SI1646 -MAKB0_SI2276 -MAKB0_SX116 -MAKB0_SX206 -MAKB0_SX26 -MAKB0_SX296 -MAKB0_SX386 -MAKR0_SI1352 -MAKR0_SI1982 -MAKR0_SI722 -MAKR0_SX182 -MAKR0_SX272 -MAKR0_SX362 -MAKR0_SX452 -MAKR0_SX92 -MAPV0_SI1293 -MAPV0_SI1923 -MAPV0_SI663 -MAPV0_SX123 -MAPV0_SX213 -MAPV0_SX303 -MAPV0_SX33 -MAPV0_SX393 -MARC0_SI1188 -MARC0_SI1818 -MARC0_SI558 -MARC0_SX108 -MARC0_SX18 -MARC0_SX198 -MARC0_SX288 -MARC0_SX378 -MARW0_SI1276 -MARW0_SI1906 -MARW0_SI646 -MARW0_SX106 -MARW0_SX16 -MARW0_SX286 -MARW0_SX349 -MARW0_SX376 -MBAR0_SI1319 -MBAR0_SI1949 -MBAR0_SI689 -MBAR0_SX149 -MBAR0_SX239 -MBAR0_SX329 -MBAR0_SX419 -MBAR0_SX59 -MBBR0_SI1055 -MBBR0_SI1685 -MBBR0_SI2315 -MBBR0_SX155 -MBBR0_SX245 -MBBR0_SX335 -MBBR0_SX425 -MBBR0_SX65 -MBCG0_SI2217 -MBCG0_SI486 -MBCG0_SI957 -MBCG0_SX147 -MBCG0_SX237 -MBCG0_SX327 -MBCG0_SX417 -MBCG0_SX57 -MBEF0_SI1281 -MBEF0_SI1911 -MBEF0_SI651 -MBEF0_SX111 -MBEF0_SX201 -MBEF0_SX21 -MBEF0_SX291 -MBEF0_SX381 -MBGT0_SI1341 -MBGT0_SI1841 -MBGT0_SI711 -MBGT0_SX171 -MBGT0_SX261 -MBGT0_SX351 -MBGT0_SX441 -MBGT0_SX81 -MBJV0_SI1247 -MBJV0_SI1877 -MBJV0_SI617 -MBJV0_SX167 -MBJV0_SX257 -MBJV0_SX347 -MBJV0_SX437 -MBJV0_SX77 -MBMA0_SI1222 -MBMA0_SI1852 -MBMA0_SI592 -MBMA0_SX142 -MBMA0_SX232 -MBMA0_SX322 -MBMA0_SX412 -MBMA0_SX52 -MBMA1_SI2207 -MBMA1_SI2214 -MBMA1_SI954 -MBMA1_SX144 -MBMA1_SX234 -MBMA1_SX324 -MBMA1_SX414 -MBMA1_SX54 -MBML0_SI1169 -MBML0_SI1799 -MBML0_SI539 -MBML0_SX179 -MBML0_SX269 -MBML0_SX359 -MBML0_SX449 -MBML0_SX89 -MBOM0_SI1014 -MBOM0_SI1644 -MBOM0_SI2274 -MBOM0_SX114 -MBOM0_SX204 -MBOM0_SX294 -MBOM0_SX311 -MBOM0_SX384 -MBSB0_SI1353 -MBSB0_SI1983 -MBSB0_SI723 -MBSB0_SX183 -MBSB0_SX273 -MBSB0_SX3 -MBSB0_SX363 -MBSB0_SX93 -MBTH0_SI2102 -MBTH0_SI505 -MBTH0_SI757 -MBTH0_SX122 -MBTH0_SX212 -MBTH0_SX302 -MBTH0_SX32 -MBTH0_SX392 -MBWP0_SI1531 -MBWP0_SI1969 -MBWP0_SI709 -MBWP0_SX169 -MBWP0_SX259 -MBWP0_SX349 -MBWP0_SX439 -MBWP0_SX79 -MCAE0_SI1447 -MCAE0_SI2077 -MCAE0_SI817 -MCAE0_SX187 -MCAE0_SX277 -MCAE0_SX367 -MCAE0_SX7 -MCAE0_SX97 -MCAL0_SI1138 -MCAL0_SI1768 -MCAL0_SI508 -MCAL0_SX148 -MCAL0_SX238 -MCAL0_SX328 -MCAL0_SX418 -MCAL0_SX58 -MCDC0_SI1292 -MCDC0_SI1922 -MCDC0_SI662 -MCDC0_SX122 -MCDC0_SX212 -MCDC0_SX302 -MCDC0_SX32 -MCDC0_SX392 -MCDD0_SI1513 -MCDD0_SI2143 -MCDD0_SI883 -MCDD0_SX163 -MCDD0_SX253 -MCDD0_SX343 -MCDD0_SX433 -MCDD0_SX73 -MCDR0_SI1154 -MCDR0_SI1784 -MCDR0_SI524 -MCDR0_SX164 -MCDR0_SX254 -MCDR0_SX344 -MCDR0_SX434 -MCDR0_SX74 -MCEF0_SI1135 -MCEF0_SI1765 -MCEF0_SI842 -MCEF0_SX145 -MCEF0_SX235 -MCEF0_SX325 -MCEF0_SX415 -MCEF0_SX55 -MCEW0_SI1442 -MCEW0_SI2072 -MCEW0_SI812 -MCEW0_SX182 -MCEW0_SX272 -MCEW0_SX362 -MCEW0_SX452 -MCEW0_SX92 -MCHL0_SI1347 -MCHL0_SI1404 -MCHL0_SI1977 -MCHL0_SX177 -MCHL0_SX267 -MCHL0_SX357 -MCHL0_SX447 -MCHL0_SX87 -MCLK0_SI1660 -MCLK0_SI2290 -MCLK0_SI650 -MCLK0_SX130 -MCLK0_SX220 -MCLK0_SX310 -MCLK0_SX40 -MCLK0_SX400 -MCLM0_SI1456 -MCLM0_SI2086 -MCLM0_SI826 -MCLM0_SX106 -MCLM0_SX16 -MCLM0_SX196 -MCLM0_SX286 -MCLM0_SX376 -MCPM0_SI1194 -MCPM0_SI1824 -MCPM0_SI564 -MCPM0_SX114 -MCPM0_SX204 -MCPM0_SX24 -MCPM0_SX294 -MCPM0_SX384 -MCRE0_SI1121 -MCRE0_SI1725 -MCRE0_SI1751 -MCRE0_SX131 -MCRE0_SX221 -MCRE0_SX24 -MCRE0_SX401 -MCRE0_SX41 -MCSS0_SI1380 -MCSS0_SI688 -MCSS0_SI750 -MCSS0_SX120 -MCSS0_SX210 -MCSS0_SX30 -MCSS0_SX300 -MCSS0_SX390 -MCTH0_SI1209 -MCTH0_SI1839 -MCTH0_SI579 -MCTH0_SX129 -MCTH0_SX219 -MCTH0_SX309 -MCTH0_SX39 -MCTH0_SX399 -MCTM0_SI1350 -MCTM0_SI1980 -MCTM0_SI720 -MCTM0_SX180 -MCTM0_SX270 -MCTM0_SX360 -MCTM0_SX450 -MCTM0_SX90 -MCXM0_SI1351 -MCXM0_SI1981 -MCXM0_SI721 -MCXM0_SX181 -MCXM0_SX271 -MCXM0_SX361 -MCXM0_SX451 -MCXM0_SX91 -MDAC0_SI1261 -MDAC0_SI1837 -MDAC0_SI631 -MDAC0_SX181 -MDAC0_SX271 -MDAC0_SX361 -MDAC0_SX451 -MDAC0_SX91 -MDAS0_SI1266 -MDAS0_SI1896 -MDAS0_SI636 -MDAS0_SX186 -MDAS0_SX21 -MDAS0_SX276 -MDAS0_SX6 -MDAS0_SX96 -MDBB1_SI1006 -MDBB1_SI1636 -MDBB1_SI2056 -MDBB1_SX106 -MDBB1_SX16 -MDBB1_SX196 -MDBB1_SX286 -MDBB1_SX376 -MDBP0_SI1158 -MDBP0_SI1788 -MDBP0_SI528 -MDBP0_SX168 -MDBP0_SX258 -MDBP0_SX348 -MDBP0_SX438 -MDBP0_SX78 -MDCD0_SI1415 -MDCD0_SI2045 -MDCD0_SI785 -MDCD0_SX155 -MDCD0_SX245 -MDCD0_SX335 -MDCD0_SX425 -MDCD0_SX65 -MDCM0_SI1480 -MDCM0_SI2110 -MDCM0_SI850 -MDCM0_SX130 -MDCM0_SX220 -MDCM0_SX310 -MDCM0_SX40 -MDCM0_SX400 -MDDC0_SI1419 -MDDC0_SI2049 -MDDC0_SI789 -MDDC0_SX159 -MDDC0_SX249 -MDDC0_SX339 -MDDC0_SX429 -MDDC0_SX69 -MDED0_SI1170 -MDED0_SI1800 -MDED0_SI540 -MDED0_SX180 -MDED0_SX270 -MDED0_SX360 -MDED0_SX450 -MDED0_SX90 -MDEF0_SI1123 -MDEF0_SI1563 -MDEF0_SI2193 -MDEF0_SX123 -MDEF0_SX213 -MDEF0_SX303 -MDEF0_SX33 -MDEF0_SX393 -MDEM0_SI1868 -MDEM0_SI608 -MDEM0_SI800 -MDEM0_SX158 -MDEM0_SX248 -MDEM0_SX338 -MDEM0_SX428 -MDEM0_SX68 -MDHL0_SI1439 -MDHL0_SI2069 -MDHL0_SI809 -MDHL0_SX179 -MDHL0_SX269 -MDHL0_SX359 -MDHL0_SX449 -MDHL0_SX89 -MDHS0_SI1530 -MDHS0_SI2160 -MDHS0_SI900 -MDHS0_SX180 -MDHS0_SX270 -MDHS0_SX360 -MDHS0_SX450 -MDHS0_SX90 -MDJM0_SI1455 -MDJM0_SI2085 -MDJM0_SI825 -MDJM0_SX105 -MDJM0_SX15 -MDJM0_SX195 -MDJM0_SX285 -MDJM0_SX375 -MDKS0_SI1066 -MDKS0_SI1696 -MDKS0_SI2326 -MDKS0_SX166 -MDKS0_SX256 -MDKS0_SX346 -MDKS0_SX436 -MDKS0_SX76 -MDLB0_SI1306 -MDLB0_SI1936 -MDLB0_SI676 -MDLB0_SX136 -MDLB0_SX226 -MDLB0_SX316 -MDLB0_SX406 -MDLB0_SX46 -MDLC0_SI1395 -MDLC0_SI2025 -MDLC0_SI765 -MDLC0_SX135 -MDLC0_SX225 -MDLC0_SX315 -MDLC0_SX405 -MDLC0_SX45 -MDLC1_SI1435 -MDLC1_SI2065 -MDLC1_SI2144 -MDLC1_SX175 -MDLC1_SX265 -MDLC1_SX355 -MDLC1_SX445 -MDLC1_SX85 -MDLC2_SI1614 -MDLC2_SI2244 -MDLC2_SI984 -MDLC2_SX174 -MDLC2_SX264 -MDLC2_SX354 -MDLC2_SX444 -MDLC2_SX84 -MDLH0_SI1960 -MDLH0_SI574 -MDLH0_SI700 -MDLH0_SX160 -MDLH0_SX250 -MDLH0_SX340 -MDLH0_SX430 -MDLH0_SX70 -MDLM0_SI1234 -MDLM0_SI1864 -MDLM0_SI604 -MDLM0_SX154 -MDLM0_SX244 -MDLM0_SX334 -MDLM0_SX424 -MDLM0_SX64 -MDLR0_SI1233 -MDLR0_SI1863 -MDLR0_SI603 -MDLR0_SX153 -MDLR0_SX243 -MDLR0_SX333 -MDLR0_SX423 -MDLR0_SX63 -MDLR1_SI1299 -MDLR1_SI1929 -MDLR1_SI669 -MDLR1_SX129 -MDLR1_SX219 -MDLR1_SX309 -MDLR1_SX39 -MDLR1_SX399 -MDMA0_SI1238 -MDMA0_SI1430 -MDMA0_SI2060 -MDMA0_SX170 -MDMA0_SX260 -MDMA0_SX350 -MDMA0_SX440 -MDMA0_SX80 -MDMT0_SI1832 -MDMT0_SI2341 -MDMT0_SI572 -MDMT0_SX122 -MDMT0_SX212 -MDMT0_SX302 -MDMT0_SX32 -MDMT0_SX392 -MDNS0_SI1011 -MDNS0_SI2271 -MDNS0_SI873 -MDNS0_SX111 -MDNS0_SX201 -MDNS0_SX21 -MDNS0_SX291 -MDNS0_SX381 -MDPB0_SI1760 -MDPB0_SI2126 -MDPB0_SI866 -MDPB0_SX146 -MDPB0_SX236 -MDPB0_SX326 -MDPB0_SX416 -MDPB0_SX56 -MDPK0_SI1053 -MDPK0_SI1683 -MDPK0_SI552 -MDPK0_SX153 -MDPK0_SX243 -MDPK0_SX333 -MDPK0_SX423 -MDPK0_SX63 -MDPS0_SI1651 -MDPS0_SI1979 -MDPS0_SI719 -MDPS0_SX179 -MDPS0_SX269 -MDPS0_SX359 -MDPS0_SX449 -MDPS0_SX89 -MDRD0_SI1382 -MDRD0_SI2012 -MDRD0_SI752 -MDRD0_SX122 -MDRD0_SX212 -MDRD0_SX302 -MDRD0_SX32 -MDRD0_SX392 -MDSJ0_SI1462 -MDSJ0_SI2092 -MDSJ0_SI832 -MDSJ0_SX112 -MDSJ0_SX22 -MDSJ0_SX292 -MDSJ0_SX382 -MDSJ0_SX438 -MDSS0_SI1881 -MDSS0_SI2087 -MDSS0_SI621 -MDSS0_SX171 -MDSS0_SX261 -MDSS0_SX351 -MDSS0_SX441 -MDSS0_SX81 -MDSS1_SI1327 -MDSS1_SI1713 -MDSS1_SI697 -MDSS1_SX157 -MDSS1_SX247 -MDSS1_SX337 -MDSS1_SX427 -MDSS1_SX67 -MDTB0_SI1200 -MDTB0_SI1830 -MDTB0_SI570 -MDTB0_SX120 -MDTB0_SX210 -MDTB0_SX300 -MDTB0_SX321 -MDTB0_SX390 -MDWD0_SI1260 -MDWD0_SI1890 -MDWD0_SI557 -MDWD0_SX180 -MDWD0_SX270 -MDWD0_SX360 -MDWD0_SX450 -MDWD0_SX90 -MDWH0_SI1168 -MDWH0_SI1925 -MDWH0_SI665 -MDWH0_SX125 -MDWH0_SX215 -MDWH0_SX305 -MDWH0_SX35 -MDWH0_SX395 -MDWM0_SI1546 -MDWM0_SI2176 -MDWM0_SI916 -MDWM0_SX106 -MDWM0_SX16 -MDWM0_SX286 -MDWM0_SX376 -MDWM0_SX433 -MEAL0_SI1547 -MEAL0_SI2177 -MEAL0_SI917 -MEAL0_SX107 -MEAL0_SX197 -MEAL0_SX287 -MEAL0_SX347 -MEAL0_SX377 -MEDR0_SI1374 -MEDR0_SI2004 -MEDR0_SI744 -MEDR0_SX114 -MEDR0_SX204 -MEDR0_SX24 -MEDR0_SX294 -MEDR0_SX384 -MEFG0_SI465 -MEFG0_SI491 -MEFG0_SI598 -MEFG0_SX105 -MEFG0_SX15 -MEFG0_SX195 -MEFG0_SX285 -MEFG0_SX375 -MEGJ0_SI1337 -MEGJ0_SI1967 -MEGJ0_SI707 -MEGJ0_SX167 -MEGJ0_SX257 -MEGJ0_SX3 -MEGJ0_SX437 -MEGJ0_SX77 -MEJL0_SI1592 -MEJL0_SI1654 -MEJL0_SI962 -MEJL0_SX152 -MEJL0_SX242 -MEJL0_SX332 -MEJL0_SX422 -MEJL0_SX62 -MEJS0_SI1240 -MEJS0_SI1870 -MEJS0_SI610 -MEJS0_SX160 -MEJS0_SX250 -MEJS0_SX340 -MEJS0_SX430 -MEJS0_SX70 -MESG0_SI1332 -MESG0_SI1962 -MESG0_SI702 -MESG0_SX162 -MESG0_SX252 -MESG0_SX342 -MESG0_SX432 -MESG0_SX72 -MESJ0_SI2039 -MESJ0_SI2257 -MESJ0_SI997 -MESJ0_SX187 -MESJ0_SX277 -MESJ0_SX367 -MESJ0_SX7 -MESJ0_SX97 -MEWM0_SI1348 -MEWM0_SI1978 -MEWM0_SI718 -MEWM0_SX178 -MEWM0_SX268 -MEWM0_SX358 -MEWM0_SX448 -MEWM0_SX88 -MFER0_SI1492 -MFER0_SI2122 -MFER0_SI862 -MFER0_SX142 -MFER0_SX232 -MFER0_SX322 -MFER0_SX412 -MFER0_SX52 -MFMC0_SI1132 -MFMC0_SI1762 -MFMC0_SI502 -MFMC0_SX142 -MFMC0_SX232 -MFMC0_SX322 -MFMC0_SX412 -MFMC0_SX52 -MFRM0_SI1155 -MFRM0_SI1717 -MFRM0_SI1785 -MFRM0_SX165 -MFRM0_SX255 -MFRM0_SX345 -MFRM0_SX435 -MFRM0_SX75 -MFWK0_SI1249 -MFWK0_SI1879 -MFWK0_SI619 -MFWK0_SX169 -MFWK0_SX259 -MFWK0_SX349 -MFWK0_SX439 -MFWK0_SX79 -MFXS0_SI1674 -MFXS0_SI2225 -MFXS0_SI2304 -MFXS0_SX144 -MFXS0_SX234 -MFXS0_SX324 -MFXS0_SX414 -MFXS0_SX54 -MFXV0_SI1005 -MFXV0_SI1342 -MFXV0_SI1635 -MFXV0_SX105 -MFXV0_SX15 -MFXV0_SX195 -MFXV0_SX285 -MFXV0_SX375 -MGAF0_SI1282 -MGAF0_SI1912 -MGAF0_SI652 -MGAF0_SX112 -MGAF0_SX202 -MGAF0_SX22 -MGAF0_SX292 -MGAF0_SX382 -MGAG0_SI1321 -MGAG0_SI645 -MGAG0_SI691 -MGAG0_SX151 -MGAG0_SX241 -MGAG0_SX331 -MGAG0_SX421 -MGAG0_SX61 -MGAK0_SI1036 -MGAK0_SI1666 -MGAK0_SI2296 -MGAK0_SX136 -MGAK0_SX226 -MGAK0_SX316 -MGAK0_SX406 -MGAK0_SX46 -MGAR0_SI1212 -MGAR0_SI1694 -MGAR0_SI1842 -MGAR0_SX132 -MGAR0_SX222 -MGAR0_SX312 -MGAR0_SX402 -MGAR0_SX42 -MGAW0_SI1165 -MGAW0_SI1802 -MGAW0_SI535 -MGAW0_SX175 -MGAW0_SX265 -MGAW0_SX355 -MGAW0_SX445 -MGAW0_SX85 -MGES0_SI1481 -MGES0_SI2111 -MGES0_SI851 -MGES0_SX131 -MGES0_SX221 -MGES0_SX311 -MGES0_SX401 -MGES0_SX41 -MGJC0_SI1256 -MGJC0_SI1335 -MGJC0_SI1965 -MGJC0_SX165 -MGJC0_SX255 -MGJC0_SX345 -MGJC0_SX435 -MGJC0_SX75 -MGRL0_SI1497 -MGRL0_SI2127 -MGRL0_SI867 -MGRL0_SX147 -MGRL0_SX237 -MGRL0_SX327 -MGRL0_SX417 -MGRL0_SX57 -MGRP0_SI1317 -MGRP0_SI1947 -MGRP0_SI687 -MGRP0_SX147 -MGRP0_SX237 -MGRP0_SX327 -MGRP0_SX417 -MGRP0_SX57 -MGSH0_SI1176 -MGSH0_SI1806 -MGSH0_SI546 -MGSH0_SX127 -MGSH0_SX186 -MGSH0_SX276 -MGSH0_SX6 -MGSH0_SX96 -MGSL0_SI1164 -MGSL0_SI534 -MGSL0_SI797 -MGSL0_SX174 -MGSL0_SX264 -MGSL0_SX354 -MGSL0_SX444 -MGSL0_SX84 -MGXP0_SI1087 -MGXP0_SI457 -MGXP0_SI525 -MGXP0_SX187 -MGXP0_SX277 -MGXP0_SX367 -MGXP0_SX7 -MGXP0_SX97 -MHBS0_SI1575 -MHBS0_SI2205 -MHBS0_SI945 -MHBS0_SX135 -MHBS0_SX225 -MHBS0_SX315 -MHBS0_SX405 -MHBS0_SX45 -MHIT0_SI1613 -MHIT0_SI2243 -MHIT0_SI983 -MHIT0_SX173 -MHIT0_SX263 -MHIT0_SX353 -MHIT0_SX443 -MHIT0_SX83 -MHJB0_SI1017 -MHJB0_SI1647 -MHJB0_SI2277 -MHJB0_SX117 -MHJB0_SX207 -MHJB0_SX27 -MHJB0_SX297 -MHJB0_SX387 -MHMG0_SI1365 -MHMG0_SI1995 -MHMG0_SI735 -MHMG0_SX105 -MHMG0_SX15 -MHMG0_SX195 -MHMG0_SX285 -MHMG0_SX375 -MHMR0_SI1119 -MHMR0_SI1692 -MHMR0_SI489 -MHMR0_SX129 -MHMR0_SX219 -MHMR0_SX309 -MHMR0_SX39 -MHMR0_SX399 -MHRM0_SI1475 -MHRM0_SI2218 -MHRM0_SI958 -MHRM0_SX148 -MHRM0_SX238 -MHRM0_SX328 -MHRM0_SX418 -MHRM0_SX58 -MHXL0_SI1772 -MHXL0_SI512 -MHXL0_SI612 -MHXL0_SX152 -MHXL0_SX242 -MHXL0_SX332 -MHXL0_SX422 -MHXL0_SX62 -MILB0_SI2163 -MILB0_SI807 -MILB0_SI903 -MILB0_SX183 -MILB0_SX273 -MILB0_SX3 -MILB0_SX363 -MILB0_SX93 -MJAC0_SI1331 -MJAC0_SI2148 -MJAC0_SI701 -MJAC0_SX251 -MJAC0_SX307 -MJAC0_SX341 -MJAC0_SX431 -MJAC0_SX71 -MJAE0_SI1524 -MJAE0_SI1999 -MJAE0_SI2154 -MJAE0_SX174 -MJAE0_SX264 -MJAE0_SX354 -MJAE0_SX444 -MJAE0_SX84 -MJAI0_SI1604 -MJAI0_SI682 -MJAI0_SI710 -MJAI0_SX164 -MJAI0_SX254 -MJAI0_SX344 -MJAI0_SX434 -MJAI0_SX74 -MJBG0_SI1232 -MJBG0_SI1724 -MJBG0_SI1862 -MJBG0_SX152 -MJBG0_SX242 -MJBG0_SX332 -MJBG0_SX422 -MJBG0_SX62 -MJDA0_SI1031 -MJDA0_SI1661 -MJDA0_SI2291 -MJDA0_SX131 -MJDA0_SX221 -MJDA0_SX311 -MJDA0_SX401 -MJDA0_SX41 -MJDC0_SI1161 -MJDC0_SI2165 -MJDC0_SI531 -MJDC0_SX171 -MJDC0_SX261 -MJDC0_SX351 -MJDC0_SX441 -MJDC0_SX81 -MJDE0_SI1120 -MJDE0_SI463 -MJDE0_SI490 -MJDE0_SX130 -MJDE0_SX220 -MJDE0_SX310 -MJDE0_SX40 -MJDE0_SX400 -MJDG0_SI1042 -MJDG0_SI1672 -MJDG0_SI1705 -MJDG0_SX142 -MJDG0_SX232 -MJDG0_SX322 -MJDG0_SX412 -MJDG0_SX52 -MJDM0_SI1340 -MJDM0_SI1937 -MJDM0_SI974 -MJDM0_SX170 -MJDM0_SX260 -MJDM0_SX350 -MJDM0_SX440 -MJDM0_SX80 -MJEB0_SI1286 -MJEB0_SI1916 -MJEB0_SI656 -MJEB0_SX170 -MJEB0_SX206 -MJEB0_SX26 -MJEB0_SX296 -MJEB0_SX386 -MJEB1_SI1467 -MJEB1_SI2097 -MJEB1_SI837 -MJEB1_SX117 -MJEB1_SX207 -MJEB1_SX27 -MJEB1_SX297 -MJEB1_SX387 -MJEE0_SI1237 -MJEE0_SI1867 -MJEE0_SI607 -MJEE0_SX157 -MJEE0_SX247 -MJEE0_SX337 -MJEE0_SX427 -MJEE0_SX67 -MJFH0_SI1107 -MJFH0_SI1737 -MJFH0_SI477 -MJFH0_SX117 -MJFH0_SX207 -MJFH0_SX27 -MJFH0_SX297 -MJFH0_SX387 -MJFR0_SI1605 -MJFR0_SI2235 -MJFR0_SI975 -MJFR0_SX165 -MJFR0_SX255 -MJFR0_SX345 -MJFR0_SX435 -MJFR0_SX75 -MJHI0_SI1328 -MJHI0_SI555 -MJHI0_SI698 -MJHI0_SX158 -MJHI0_SX248 -MJHI0_SX338 -MJHI0_SX428 -MJHI0_SX68 -MJJB0_SI1139 -MJJB0_SI1277 -MJJB0_SI1769 -MJJB0_SX149 -MJJB0_SX239 -MJJB0_SX329 -MJJB0_SX419 -MJJB0_SX59 -MJJJ0_SI1163 -MJJJ0_SI1793 -MJJJ0_SI533 -MJJJ0_SX173 -MJJJ0_SX263 -MJJJ0_SX353 -MJJJ0_SX443 -MJJJ0_SX83 -MJJM0_SI1251 -MJJM0_SI1457 -MJJM0_SI827 -MJJM0_SX107 -MJJM0_SX17 -MJJM0_SX197 -MJJM0_SX287 -MJJM0_SX377 -MJKR0_SI1201 -MJKR0_SI1831 -MJKR0_SI571 -MJKR0_SX121 -MJKR0_SX211 -MJKR0_SX301 -MJKR0_SX31 -MJKR0_SX391 -MJLB0_SI1616 -MJLB0_SI2246 -MJLB0_SI986 -MJLB0_SX176 -MJLB0_SX266 -MJLB0_SX356 -MJLB0_SX446 -MJLB0_SX86 -MJLG1_SI1012 -MJLG1_SI1642 -MJLG1_SI2272 -MJLG1_SX112 -MJLG1_SX202 -MJLG1_SX22 -MJLG1_SX292 -MJLG1_SX382 -MJLS0_SI1096 -MJLS0_SI1726 -MJLS0_SI466 -MJLS0_SX106 -MJLS0_SX16 -MJLS0_SX196 -MJLS0_SX286 -MJLS0_SX376 -MJMA0_SI1495 -MJMA0_SI2125 -MJMA0_SI865 -MJMA0_SX145 -MJMA0_SX235 -MJMA0_SX325 -MJMA0_SX415 -MJMA0_SX55 -MJMD0_SI1028 -MJMD0_SI1658 -MJMD0_SI2288 -MJMD0_SX128 -MJMD0_SX218 -MJMD0_SX308 -MJMD0_SX38 -MJMD0_SX398 -MJMM0_SI1255 -MJMM0_SI1885 -MJMM0_SI625 -MJMM0_SX175 -MJMM0_SX265 -MJMM0_SX355 -MJMM0_SX445 -MJMM0_SX85 -MJPG0_SI1191 -MJPG0_SI1821 -MJPG0_SI561 -MJPG0_SX111 -MJPG0_SX201 -MJPG0_SX21 -MJPG0_SX291 -MJPG0_SX381 -MJPM0_SI1368 -MJPM0_SI1998 -MJPM0_SI738 -MJPM0_SX108 -MJPM0_SX18 -MJPM0_SX198 -MJPM0_SX288 -MJPM0_SX378 -MJPM1_SI1897 -MJPM1_SI2280 -MJPM1_SI761 -MJPM1_SX131 -MJPM1_SX221 -MJPM1_SX311 -MJPM1_SX401 -MJPM1_SX41 -MJRA0_SI1236 -MJRA0_SI1866 -MJRA0_SI606 -MJRA0_SX156 -MJRA0_SX246 -MJRA0_SX336 -MJRA0_SX426 -MJRA0_SX66 -MJRG0_SI1366 -MJRG0_SI1996 -MJRG0_SI736 -MJRG0_SX106 -MJRG0_SX16 -MJRG0_SX286 -MJRG0_SX352 -MJRG0_SX376 -MJRH0_SI1125 -MJRH0_SI1755 -MJRH0_SI1840 -MJRH0_SX135 -MJRH0_SX225 -MJRH0_SX315 -MJRH0_SX405 -MJRH0_SX45 -MJRH1_SI1558 -MJRH1_SI1774 -MJRH1_SI514 -MJRH1_SX154 -MJRH1_SX244 -MJRH1_SX334 -MJRH1_SX424 -MJRH1_SX64 -MJRK0_SI1662 -MJRK0_SI2103 -MJRK0_SI880 -MJRK0_SX160 -MJRK0_SX250 -MJRK0_SX340 -MJRK0_SX430 -MJRK0_SX70 -MJRP0_SI1835 -MJRP0_SI1845 -MJRP0_SI585 -MJRP0_SX135 -MJRP0_SX225 -MJRP0_SX315 -MJRP0_SX405 -MJRP0_SX45 -MJSR0_SI1424 -MJSR0_SI2054 -MJSR0_SI794 -MJSR0_SX164 -MJSR0_SX254 -MJSR0_SX344 -MJSR0_SX434 -MJSR0_SX74 -MJWG0_SI2155 -MJWG0_SI813 -MJWG0_SI895 -MJWG0_SX175 -MJWG0_SX265 -MJWG0_SX355 -MJWG0_SX445 -MJWG0_SX85 -MJWS0_SI1143 -MJWS0_SI1773 -MJWS0_SI513 -MJWS0_SX153 -MJWS0_SX243 -MJWS0_SX333 -MJWS0_SX423 -MJWS0_SX63 -MJWT0_SI1291 -MJWT0_SI1381 -MJWT0_SI751 -MJWT0_SX121 -MJWT0_SX211 -MJWT0_SX301 -MJWT0_SX31 -MJWT0_SX391 -MJXA0_SI1507 -MJXA0_SI2137 -MJXA0_SI877 -MJXA0_SX157 -MJXA0_SX247 -MJXA0_SX337 -MJXA0_SX427 -MJXA0_SX67 -MJXL0_SI1172 -MJXL0_SI1795 -MJXL0_SI542 -MJXL0_SX182 -MJXL0_SX272 -MJXL0_SX362 -MJXL0_SX452 -MJXL0_SX92 -MKAG0_SI1609 -MKAG0_SI2239 -MKAG0_SI979 -MKAG0_SX169 -MKAG0_SX259 -MKAG0_SX30 -MKAG0_SX439 -MKAG0_SX79 -MKAH0_SI1528 -MKAH0_SI2158 -MKAH0_SI898 -MKAH0_SX178 -MKAH0_SX268 -MKAH0_SX358 -MKAH0_SX448 -MKAH0_SX88 -MKAJ0_SI1414 -MKAJ0_SI2044 -MKAJ0_SI784 -MKAJ0_SX154 -MKAJ0_SX244 -MKAJ0_SX334 -MKAJ0_SX424 -MKAJ0_SX64 -MKAM0_SI1250 -MKAM0_SI1316 -MKAM0_SI1465 -MKAM0_SX146 -MKAM0_SX236 -MKAM0_SX326 -MKAM0_SX416 -MKAM0_SX56 -MKDB0_SI2132 -MKDB0_SI588 -MKDB0_SI872 -MKDB0_SX152 -MKDB0_SX242 -MKDB0_SX332 -MKDB0_SX422 -MKDB0_SX62 -MKDD0_SI1567 -MKDD0_SI2197 -MKDD0_SI937 -MKDD0_SX127 -MKDD0_SX217 -MKDD0_SX307 -MKDD0_SX37 -MKDD0_SX397 -MKDT0_SI2153 -MKDT0_SI814 -MKDT0_SI893 -MKDT0_SX173 -MKDT0_SX263 -MKDT0_SX353 -MKDT0_SX443 -MKDT0_SX83 -MKES0_SI1253 -MKES0_SI1883 -MKES0_SI623 -MKES0_SX173 -MKES0_SX263 -MKES0_SX353 -MKES0_SX443 -MKES0_SX83 -MKJO0_SI1517 -MKJO0_SI2147 -MKJO0_SI887 -MKJO0_SX167 -MKJO0_SX257 -MKJO0_SX424 -MKJO0_SX437 -MKJO0_SX77 -MKLN0_SI1598 -MKLN0_SI2228 -MKLN0_SI968 -MKLN0_SX158 -MKLN0_SX248 -MKLN0_SX338 -MKLN0_SX428 -MKLN0_SX68 -MKLR0_SI1059 -MKLR0_SI1689 -MKLR0_SI2319 -MKLR0_SX159 -MKLR0_SX249 -MKLR0_SX339 -MKLR0_SX429 -MKLR0_SX69 -MKLS0_SI1437 -MKLS0_SI1533 -MKLS0_SI2067 -MKLS0_SX177 -MKLS0_SX267 -MKLS0_SX357 -MKLS0_SX447 -MKLS0_SX87 -MKLS1_SI1545 -MKLS1_SI2175 -MKLS1_SI915 -MKLS1_SX105 -MKLS1_SX15 -MKLS1_SX195 -MKLS1_SX285 -MKLS1_SX375 -MKLW0_SI1571 -MKLW0_SI1844 -MKLW0_SI2201 -MKLW0_SX131 -MKLW0_SX221 -MKLW0_SX311 -MKLW0_SX401 -MKLW0_SX41 -MKRG0_SI1491 -MKRG0_SI2121 -MKRG0_SI861 -MKRG0_SX141 -MKRG0_SX231 -MKRG0_SX31 -MKRG0_SX411 -MKRG0_SX51 -MKXL0_SI1185 -MKXL0_SI1815 -MKXL0_SI1958 -MKXL0_SX105 -MKXL0_SX15 -MKXL0_SX195 -MKXL0_SX285 -MKXL0_SX375 -MLBC0_SI1239 -MLBC0_SI1869 -MLBC0_SI609 -MLBC0_SX159 -MLBC0_SX249 -MLBC0_SX339 -MLBC0_SX429 -MLBC0_SX69 -MLEL0_SI1246 -MLEL0_SI1876 -MLEL0_SI616 -MLEL0_SX166 -MLEL0_SX256 -MLEL0_SX346 -MLEL0_SX436 -MLEL0_SX76 -MLJC0_SI1225 -MLJC0_SI1855 -MLJC0_SI595 -MLJC0_SX145 -MLJC0_SX235 -MLJC0_SX325 -MLJC0_SX415 -MLJC0_SX55 -MLJH0_SI1324 -MLJH0_SI1422 -MLJH0_SI694 -MLJH0_SX154 -MLJH0_SX244 -MLJH0_SX334 -MLJH0_SX424 -MLJH0_SX64 -MLNS0_SI1407 -MLNS0_SI2037 -MLNS0_SI777 -MLNS0_SX147 -MLNS0_SX237 -MLNS0_SX327 -MLNS0_SX417 -MLNS0_SX57 -MLSH0_SI1417 -MLSH0_SI2047 -MLSH0_SI787 -MLSH0_SX157 -MLSH0_SX247 -MLSH0_SX337 -MLSH0_SX427 -MLSH0_SX67 -MMAA0_SI1588 -MMAA0_SI2105 -MMAA0_SI845 -MMAA0_SX125 -MMAA0_SX215 -MMAA0_SX305 -MMAA0_SX35 -MMAA0_SX395 -MMAB1_SI1494 -MMAB1_SI2124 -MMAB1_SI864 -MMAB1_SX144 -MMAB1_SX234 -MMAB1_SX324 -MMAB1_SX414 -MMAB1_SX54 -MMAG0_SI1126 -MMAG0_SI1756 -MMAG0_SI496 -MMAG0_SX136 -MMAG0_SX226 -MMAG0_SX316 -MMAG0_SX406 -MMAG0_SX46 -MMAM0_SI1597 -MMAM0_SI1668 -MMAM0_SI2227 -MMAM0_SX157 -MMAM0_SX247 -MMAM0_SX337 -MMAM0_SX427 -MMAM0_SX67 -MMAR0_SI1336 -MMAR0_SI1966 -MMAR0_SI706 -MMAR0_SX166 -MMAR0_SX256 -MMAR0_SX346 -MMAR0_SX436 -MMAR0_SX76 -MMBS0_SI1151 -MMBS0_SI1781 -MMBS0_SI521 -MMBS0_SX161 -MMBS0_SX251 -MMBS0_SX341 -MMBS0_SX431 -MMBS0_SX71 -MMCC0_SI1338 -MMCC0_SI1968 -MMCC0_SI708 -MMCC0_SX168 -MMCC0_SX258 -MMCC0_SX348 -MMCC0_SX438 -MMCC0_SX78 -MMDB0_SI1358 -MMDB0_SI1617 -MMDB0_SI987 -MMDB0_SX177 -MMDB0_SX267 -MMDB0_SX357 -MMDB0_SX447 -MMDB0_SX87 -MMDG0_SI1780 -MMDG0_SI2035 -MMDG0_SI520 -MMDG0_SX160 -MMDG0_SX250 -MMDG0_SX340 -MMDG0_SX430 -MMDG0_SX70 -MMDM0_SI1311 -MMDM0_SI1941 -MMDM0_SI681 -MMDM0_SX141 -MMDM0_SX231 -MMDM0_SX321 -MMDM0_SX411 -MMDM0_SX51 -MMDM1_SI1650 -MMDM1_SI2043 -MMDM1_SI783 -MMDM1_SX153 -MMDM1_SX243 -MMDM1_SX333 -MMDM1_SX423 -MMDM1_SX63 -MMDS0_SI1343 -MMDS0_SI1973 -MMDS0_SI713 -MMDS0_SX173 -MMDS0_SX263 -MMDS0_SX353 -MMDS0_SX443 -MMDS0_SX83 -MMEA0_SI1388 -MMEA0_SI2018 -MMEA0_SI758 -MMEA0_SX128 -MMEA0_SX218 -MMEA0_SX308 -MMEA0_SX38 -MMEA0_SX398 -MMEB0_SI1357 -MMEB0_SI1987 -MMEB0_SI727 -MMEB0_SX187 -MMEB0_SX327 -MMEB0_SX367 -MMEB0_SX7 -MMEB0_SX97 -MMGC0_SI1305 -MMGC0_SI1935 -MMGC0_SI2184 -MMGC0_SX135 -MMGC0_SX225 -MMGC0_SX315 -MMGC0_SX405 -MMGC0_SX45 -MMGG0_SI1079 -MMGG0_SI1709 -MMGG0_SI2339 -MMGG0_SX179 -MMGG0_SX269 -MMGG0_SX359 -MMGG0_SX449 -MMGG0_SX89 -MMGK0_SI1322 -MMGK0_SI1952 -MMGK0_SI692 -MMGK0_SX152 -MMGK0_SX242 -MMGK0_SX332 -MMGK0_SX422 -MMGK0_SX62 -MMJB1_SI1408 -MMJB1_SI2038 -MMJB1_SI778 -MMJB1_SX148 -MMJB1_SX238 -MMJB1_SX328 -MMJB1_SX418 -MMJB1_SX58 -MMLM0_SI1527 -MMLM0_SI2150 -MMLM0_SI897 -MMLM0_SX177 -MMLM0_SX267 -MMLM0_SX357 -MMLM0_SX447 -MMLM0_SX87 -MMPM0_SI1061 -MMPM0_SI1691 -MMPM0_SI2321 -MMPM0_SX161 -MMPM0_SX251 -MMPM0_SX341 -MMPM0_SX431 -MMPM0_SX71 -MMRP0_SI2034 -MMRP0_SI717 -MMRP0_SI774 -MMRP0_SX144 -MMRP0_SX234 -MMRP0_SX324 -MMRP0_SX414 -MMRP0_SX54 -MMSM0_SI1106 -MMSM0_SI1736 -MMSM0_SI476 -MMSM0_SX116 -MMSM0_SX206 -MMSM0_SX26 -MMSM0_SX296 -MMSM0_SX386 -MMVP0_SI1284 -MMVP0_SI1914 -MMVP0_SI654 -MMVP0_SX114 -MMVP0_SX204 -MMVP0_SX294 -MMVP0_SX347 -MMVP0_SX384 -MMWB0_SI1619 -MMWB0_SI2249 -MMWB0_SI989 -MMWB0_SX179 -MMWB0_SX269 -MMWB0_SX359 -MMWB0_SX449 -MMWB0_SX89 -MMWS0_SI1518 -MMWS0_SI559 -MMWS0_SI888 -MMWS0_SX168 -MMWS0_SX258 -MMWS0_SX348 -MMWS0_SX438 -MMWS0_SX78 -MMWS1_SI1071 -MMWS1_SI1701 -MMWS1_SI2331 -MMWS1_SX261 -MMWS1_SX27 -MMWS1_SX351 -MMWS1_SX441 -MMWS1_SX81 -MMXS0_SI2136 -MMXS0_SI629 -MMXS0_SI876 -MMXS0_SX156 -MMXS0_SX246 -MMXS0_SX336 -MMXS0_SX426 -MMXS0_SX66 -MNET0_SI1446 -MNET0_SI2076 -MNET0_SI816 -MNET0_SX186 -MNET0_SX276 -MNET0_SX366 -MNET0_SX6 -MNET0_SX96 -MNTW0_SI1068 -MNTW0_SI1698 -MNTW0_SI2328 -MNTW0_SX168 -MNTW0_SX202 -MNTW0_SX258 -MNTW0_SX348 -MNTW0_SX78 -MPAR0_SI1576 -MPAR0_SI2206 -MPAR0_SI946 -MPAR0_SX136 -MPAR0_SX226 -MPAR0_SX316 -MPAR0_SX406 -MPAR0_SX46 -MPEB0_SI1034 -MPEB0_SI1860 -MPEB0_SI600 -MPEB0_SX150 -MPEB0_SX240 -MPEB0_SX330 -MPEB0_SX420 -MPEB0_SX60 -MPFU0_SI1258 -MPFU0_SI1888 -MPFU0_SI628 -MPFU0_SX178 -MPFU0_SX268 -MPFU0_SX358 -MPFU0_SX448 -MPFU0_SX88 -MPGH0_SI1554 -MPGH0_SI675 -MPGH0_SI924 -MPGH0_SX114 -MPGH0_SX204 -MPGH0_SX24 -MPGH0_SX294 -MPGH0_SX384 -MPGR0_SI1410 -MPGR0_SI2040 -MPGR0_SI780 -MPGR0_SX150 -MPGR0_SX240 -MPGR0_SX330 -MPGR0_SX420 -MPGR0_SX60 -MPGR1_SI1269 -MPGR1_SI1499 -MPGR1_SI2129 -MPGR1_SX149 -MPGR1_SX239 -MPGR1_SX329 -MPGR1_SX419 -MPGR1_SX59 -MPMB0_SI1501 -MPMB0_SI2131 -MPMB0_SI871 -MPMB0_SX151 -MPMB0_SX241 -MPMB0_SX331 -MPMB0_SX421 -MPMB0_SX61 -MPPC0_SI1412 -MPPC0_SI2042 -MPPC0_SI782 -MPPC0_SX152 -MPPC0_SX242 -MPPC0_SX332 -MPPC0_SX422 -MPPC0_SX62 -MPRB0_SI1205 -MPRB0_SI1215 -MPRB0_SI575 -MPRB0_SX125 -MPRB0_SX215 -MPRB0_SX305 -MPRB0_SX35 -MPRB0_SX395 -MPRD0_SI1431 -MPRD0_SI2061 -MPRD0_SI801 -MPRD0_SX171 -MPRD0_SX261 -MPRD0_SX351 -MPRD0_SX441 -MPRD0_SX81 -MPRK0_SI1097 -MPRK0_SI1727 -MPRK0_SI467 -MPRK0_SX107 -MPRK0_SX17 -MPRK0_SX197 -MPRK0_SX287 -MPRK0_SX377 -MPRT0_SI1210 -MPRT0_SI495 -MPRT0_SI580 -MPRT0_SX130 -MPRT0_SX220 -MPRT0_SX310 -MPRT0_SX40 -MPRT0_SX400 -MPSW0_SI1067 -MPSW0_SI1697 -MPSW0_SI2327 -MPSW0_SX167 -MPSW0_SX24 -MPSW0_SX257 -MPSW0_SX437 -MPSW0_SX77 -MRAB0_SI1224 -MRAB0_SI1854 -MRAB0_SI594 -MRAB0_SX144 -MRAB0_SX234 -MRAB0_SX324 -MRAB0_SX414 -MRAB0_SX54 -MRAB1_SI1478 -MRAB1_SI2108 -MRAB1_SI848 -MRAB1_SX128 -MRAB1_SX218 -MRAB1_SX308 -MRAB1_SX38 -MRAB1_SX398 -MRAI0_SI1954 -MRAI0_SI2052 -MRAI0_SI792 -MRAI0_SX162 -MRAI0_SX252 -MRAI0_SX342 -MRAI0_SX432 -MRAI0_SX72 -MRAM0_SI1275 -MRAM0_SI1905 -MRAM0_SI1951 -MRAM0_SX105 -MRAM0_SX15 -MRAM0_SX195 -MRAM0_SX285 -MRAM0_SX375 -MRAV0_SI1008 -MRAV0_SI1638 -MRAV0_SI2268 -MRAV0_SX108 -MRAV0_SX18 -MRAV0_SX198 -MRAV0_SX288 -MRAV0_SX378 -MRBC0_SI1665 -MRBC0_SI1859 -MRBC0_SI599 -MRBC0_SX149 -MRBC0_SX239 -MRBC0_SX329 -MRBC0_SX419 -MRBC0_SX59 -MRCG0_SI1428 -MRCG0_SI2058 -MRCG0_SI798 -MRCG0_SX168 -MRCG0_SX258 -MRCG0_SX348 -MRCG0_SX438 -MRCG0_SX78 -MRCW0_SI1371 -MRCW0_SI2001 -MRCW0_SI741 -MRCW0_SX111 -MRCW0_SX201 -MRCW0_SX21 -MRCW0_SX291 -MRCW0_SX381 -MRDD0_SI1050 -MRDD0_SI1680 -MRDD0_SI2310 -MRDD0_SX150 -MRDD0_SX240 -MRDD0_SX277 -MRDD0_SX330 -MRDD0_SX60 -MRDM0_SI1044 -MRDM0_SI1595 -MRDM0_SI965 -MRDM0_SX155 -MRDM0_SX245 -MRDM0_SX335 -MRDM0_SX425 -MRDM0_SX65 -MRDS0_SI1167 -MRDS0_SI1797 -MRDS0_SI537 -MRDS0_SX177 -MRDS0_SX267 -MRDS0_SX357 -MRDS0_SX447 -MRDS0_SX87 -MREE0_SI1104 -MREE0_SI1734 -MREE0_SI1959 -MREE0_SX114 -MREE0_SX204 -MREE0_SX24 -MREE0_SX294 -MREE0_SX384 -MREH1_SI1599 -MREH1_SI2229 -MREH1_SI969 -MREH1_SX159 -MREH1_SX249 -MREH1_SX339 -MREH1_SX429 -MREH1_SX69 -MREM0_SI1591 -MREM0_SI511 -MREM0_SI961 -MREM0_SX151 -MREM0_SX241 -MREM0_SX331 -MREM0_SX421 -MREM0_SX61 -MREW1_SI1500 -MREW1_SI2130 -MREW1_SI870 -MREW1_SX150 -MREW1_SX240 -MREW1_SX330 -MREW1_SX420 -MREW1_SX60 -MRFK0_SI1076 -MRFK0_SI1706 -MRFK0_SI2336 -MRFK0_SX176 -MRFK0_SX266 -MRFK0_SX356 -MRFK0_SX446 -MRFK0_SX86 -MRFL0_SI1156 -MRFL0_SI1786 -MRFL0_SI526 -MRFL0_SX166 -MRFL0_SX256 -MRFL0_SX346 -MRFL0_SX436 -MRFL0_SX76 -MRGM0_SI1162 -MRGM0_SI1792 -MRGM0_SI532 -MRGM0_SX172 -MRGM0_SX262 -MRGM0_SX416 -MRGM0_SX442 -MRGM0_SX82 -MRGS0_SI1356 -MRGS0_SI1986 -MRGS0_SI726 -MRGS0_SX186 -MRGS0_SX276 -MRGS0_SX366 -MRGS0_SX6 -MRGS0_SX96 -MRHL0_SI1515 -MRHL0_SI2145 -MRHL0_SI885 -MRHL0_SX165 -MRHL0_SX255 -MRHL0_SX345 -MRHL0_SX435 -MRHL0_SX75 -MRJB1_SI1020 -MRJB1_SI1413 -MRJB1_SI2021 -MRJB1_SX120 -MRJB1_SX210 -MRJB1_SX30 -MRJB1_SX300 -MRJB1_SX390 -MRJH0_SI1519 -MRJH0_SI889 -MRJH0_SI914 -MRJH0_SX169 -MRJH0_SX259 -MRJH0_SX307 -MRJH0_SX439 -MRJH0_SX79 -MRJM0_SI1095 -MRJM0_SI1228 -MRJM0_SI1858 -MRJM0_SX148 -MRJM0_SX238 -MRJM0_SX328 -MRJM0_SX418 -MRJM0_SX58 -MRJM1_SI1298 -MRJM1_SI1928 -MRJM1_SI668 -MRJM1_SX128 -MRJM1_SX218 -MRJM1_SX308 -MRJM1_SX38 -MRJM1_SX398 -MRJT0_SI1498 -MRJT0_SI1805 -MRJT0_SI868 -MRJT0_SX148 -MRJT0_SX238 -MRJT0_SX328 -MRJT0_SX418 -MRJT0_SX58 -MRKM0_SI1267 -MRKM0_SI1391 -MRKM0_SI637 -MRKM0_SX187 -MRKM0_SX277 -MRKM0_SX367 -MRKM0_SX7 -MRKM0_SX97 -MRLD0_SI1594 -MRLD0_SI2224 -MRLD0_SI964 -MRLD0_SX154 -MRLD0_SX244 -MRLD0_SX334 -MRLD0_SX424 -MRLD0_SX64 -MRLJ0_SI1420 -MRLJ0_SI2050 -MRLJ0_SI790 -MRLJ0_SX160 -MRLJ0_SX250 -MRLJ0_SX340 -MRLJ0_SX430 -MRLJ0_SX70 -MRLJ1_SI1671 -MRLJ1_SI2301 -MRLJ1_SI2332 -MRLJ1_SX141 -MRLJ1_SX231 -MRLJ1_SX321 -MRLJ1_SX411 -MRLJ1_SX51 -MRLK0_SI1468 -MRLK0_SI2140 -MRLK0_SI843 -MRLK0_SX123 -MRLK0_SX213 -MRLK0_SX303 -MRLK0_SX33 -MRLK0_SX393 -MRLR0_SI1196 -MRLR0_SI1826 -MRLR0_SI566 -MRLR0_SX116 -MRLR0_SX206 -MRLR0_SX26 -MRLR0_SX296 -MRLR0_SX386 -MRMB0_SI1581 -MRMB0_SI2211 -MRMB0_SI951 -MRMB0_SX141 -MRMB0_SX231 -MRMB0_SX321 -MRMB0_SX411 -MRMB0_SX51 -MRMG0_SI1080 -MRMG0_SI1710 -MRMG0_SI2340 -MRMG0_SX180 -MRMG0_SX270 -MRMG0_SX360 -MRMG0_SX450 -MRMG0_SX90 -MRMH0_SI1021 -MRMH0_SI1349 -MRMH0_SI2281 -MRMH0_SX121 -MRMH0_SX211 -MRMH0_SX301 -MRMH0_SX31 -MRMH0_SX391 -MRML0_SI1421 -MRML0_SI2051 -MRML0_SI791 -MRML0_SX161 -MRML0_SX251 -MRML0_SX341 -MRML0_SX431 -MRML0_SX71 -MRMS0_SI1113 -MRMS0_SI2057 -MRMS0_SI2100 -MRMS0_SX120 -MRMS0_SX210 -MRMS0_SX30 -MRMS0_SX300 -MRMS0_SX390 -MRPC1_SI1482 -MRPC1_SI2026 -MRPC1_SI2112 -MRPC1_SX132 -MRPC1_SX222 -MRPC1_SX312 -MRPC1_SX402 -MRPC1_SX42 -MRRE0_SI1334 -MRRE0_SI704 -MRRE0_SI952 -MRRE0_SX164 -MRRE0_SX254 -MRRE0_SX344 -MRRE0_SX434 -MRRE0_SX74 -MRSO0_SI1206 -MRSO0_SI1659 -MRSO0_SI2289 -MRSO0_SX129 -MRSO0_SX219 -MRSO0_SX309 -MRSO0_SX39 -MRSO0_SX399 -MRSP0_SI1429 -MRSP0_SI2059 -MRSP0_SI799 -MRSP0_SX169 -MRSP0_SX196 -MRSP0_SX259 -MRSP0_SX439 -MRSP0_SX79 -MRTC0_SI1458 -MRTC0_SI2088 -MRTC0_SI828 -MRTC0_SX108 -MRTC0_SX18 -MRTC0_SX198 -MRTC0_SX288 -MRTC0_SX378 -MRTJ0_SI1551 -MRTJ0_SI2032 -MRTJ0_SI772 -MRTJ0_SX142 -MRTJ0_SX232 -MRTJ0_SX322 -MRTJ0_SX412 -MRTJ0_SX52 -MRVG0_SI1140 -MRVG0_SI1770 -MRVG0_SI510 -MRVG0_SX150 -MRVG0_SX240 -MRVG0_SX330 -MRVG0_SX420 -MRVG0_SX60 -MRWA0_SI1603 -MRWA0_SI2233 -MRWA0_SI973 -MRWA0_SX163 -MRWA0_SX253 -MRWA0_SX343 -MRWA0_SX433 -MRWA0_SX73 -MRWS0_SI1102 -MRWS0_SI1732 -MRWS0_SI472 -MRWS0_SX112 -MRWS0_SX202 -MRWS0_SX22 -MRWS0_SX292 -MRWS0_SX382 -MRXB0_SI1585 -MRXB0_SI2215 -MRXB0_SI955 -MRXB0_SX145 -MRXB0_SX235 -MRXB0_SX325 -MRXB0_SX415 -MRXB0_SX55 -MSAH1_SI1049 -MSAH1_SI1679 -MSAH1_SI2309 -MSAH1_SX149 -MSAH1_SX239 -MSAH1_SX329 -MSAH1_SX419 -MSAH1_SX59 -MSAS0_SI1376 -MSAS0_SI2006 -MSAS0_SI746 -MSAS0_SX116 -MSAS0_SX206 -MSAS0_SX26 -MSAS0_SX296 -MSAS0_SX386 -MSAT0_SI1526 -MSAT0_SI2156 -MSAT0_SI896 -MSAT0_SX176 -MSAT0_SX266 -MSAT0_SX356 -MSAT0_SX446 -MSAT0_SX86 -MSAT1_SI1073 -MSAT1_SI1703 -MSAT1_SI2333 -MSAT1_SX173 -MSAT1_SX263 -MSAT1_SX353 -MSAT1_SX443 -MSAT1_SX83 -MSDB0_SI1007 -MSDB0_SI1637 -MSDB0_SI2267 -MSDB0_SX107 -MSDB0_SX17 -MSDB0_SX197 -MSDB0_SX287 -MSDB0_SX377 -MSDH0_SI2113 -MSDH0_SI2240 -MSDH0_SI980 -MSDH0_SX170 -MSDH0_SX260 -MSDH0_SX350 -MSDH0_SX440 -MSDH0_SX80 -MSDS0_SI1077 -MSDS0_SI1707 -MSDS0_SI2337 -MSDS0_SX177 -MSDS0_SX267 -MSDS0_SX357 -MSDS0_SX447 -MSDS0_SX87 -MSEM1_SI1440 -MSEM1_SI2070 -MSEM1_SI810 -MSEM1_SX180 -MSEM1_SX270 -MSEM1_SX360 -MSEM1_SX450 -MSEM1_SX90 -MSES0_SI1589 -MSES0_SI2216 -MSES0_SI2219 -MSES0_SX149 -MSES0_SX239 -MSES0_SX329 -MSES0_SX419 -MSES0_SX59 -MSFH0_SI1216 -MSFH0_SI1738 -MSFH0_SI586 -MSFH0_SX136 -MSFH0_SX226 -MSFH0_SX316 -MSFH0_SX406 -MSFH0_SX46 -MSFV0_SI1262 -MSFV0_SI1892 -MSFV0_SI632 -MSFV0_SX182 -MSFV0_SX272 -MSFV0_SX362 -MSFV0_SX452 -MSFV0_SX92 -MSJK0_SI1596 -MSJK0_SI2226 -MSJK0_SI966 -MSJK0_SX156 -MSJK0_SX246 -MSJK0_SX336 -MSJK0_SX426 -MSJK0_SX66 -MSMC0_SI1907 -MSMC0_SI509 -MSMC0_SI647 -MSMC0_SX107 -MSMC0_SX17 -MSMC0_SX197 -MSMC0_SX287 -MSMC0_SX377 -MSMR0_SI1150 -MSMR0_SI1405 -MSMR0_SI775 -MSMR0_SX145 -MSMR0_SX235 -MSMR0_SX325 -MSMR0_SX415 -MSMR0_SX55 -MSMS0_SI1433 -MSMS0_SI2063 -MSMS0_SI803 -MSMS0_SX173 -MSMS0_SX263 -MSMS0_SX353 -MSMS0_SX443 -MSMS0_SX83 -MSRG0_SI1221 -MSRG0_SI1851 -MSRG0_SI591 -MSRG0_SX141 -MSRG0_SX231 -MSRG0_SX321 -MSRG0_SX411 -MSRG0_SX51 -MSRR0_SI1131 -MSRR0_SI1761 -MSRR0_SI501 -MSRR0_SX141 -MSRR0_SX231 -MSRR0_SX30 -MSRR0_SX411 -MSRR0_SX51 -MSTF0_SI1396 -MSTF0_SI766 -MSTF0_SI852 -MSTF0_SX136 -MSTF0_SX226 -MSTF0_SX316 -MSTF0_SX406 -MSTF0_SX46 -MSVS0_SI1568 -MSVS0_SI2198 -MSVS0_SI938 -MSVS0_SX128 -MSVS0_SX218 -MSVS0_SX308 -MSVS0_SX38 -MSVS0_SX398 -MTAB0_SI1572 -MTAB0_SI2202 -MTAB0_SI942 -MTAB0_SX132 -MTAB0_SX222 -MTAB0_SX312 -MTAB0_SX402 -MTAB0_SX42 -MTAS0_SI1385 -MTAS0_SI2015 -MTAS0_SI755 -MTAS0_SX125 -MTAS0_SX215 -MTAS0_SX305 -MTAS0_SX35 -MTAS0_SX395 -MTAT0_SI1110 -MTAT0_SI1740 -MTAT0_SI811 -MTAT0_SX120 -MTAT0_SX210 -MTAT0_SX30 -MTAT0_SX300 -MTAT0_SX390 -MTAT1_SI1409 -MTAT1_SI1627 -MTAT1_SI779 -MTAT1_SX149 -MTAT1_SX239 -MTAT1_SX329 -MTAT1_SX419 -MTAT1_SX59 -MTBC0_SI1173 -MTBC0_SI1803 -MTBC0_SI543 -MTBC0_SX183 -MTBC0_SX273 -MTBC0_SX347 -MTBC0_SX363 -MTBC0_SX93 -MTCS0_SI1972 -MTCS0_SI2265 -MTCS0_SI712 -MTCS0_SX172 -MTCS0_SX262 -MTCS0_SX352 -MTCS0_SX442 -MTCS0_SX82 -MTDB0_SI1401 -MTDB0_SI2031 -MTDB0_SI771 -MTDB0_SX141 -MTDB0_SX231 -MTDB0_SX321 -MTDB0_SX411 -MTDB0_SX51 -MTDP0_SI1274 -MTDP0_SI1521 -MTDP0_SI2151 -MTDP0_SX171 -MTDP0_SX261 -MTDP0_SX351 -MTDP0_SX441 -MTDP0_SX81 -MTER0_SI1157 -MTER0_SI1787 -MTER0_SI527 -MTER0_SX167 -MTER0_SX17 -MTER0_SX257 -MTER0_SX437 -MTER0_SX77 -MTJG0_SI1520 -MTJG0_SI2157 -MTJG0_SI890 -MTJG0_SX170 -MTJG0_SX260 -MTJG0_SX350 -MTJG0_SX440 -MTJG0_SX80 -MTJM0_SI1226 -MTJM0_SI1856 -MTJM0_SI655 -MTJM0_SX146 -MTJM0_SX236 -MTJM0_SX326 -MTJM0_SX416 -MTJM0_SX56 -MTJS0_SI1192 -MTJS0_SI1822 -MTJS0_SI562 -MTJS0_SX112 -MTJS0_SX202 -MTJS0_SX22 -MTJS0_SX292 -MTJS0_SX382 -MTJU0_SI2020 -MTJU0_SI2269 -MTJU0_SI760 -MTJU0_SX130 -MTJU0_SX220 -MTJU0_SX310 -MTJU0_SX40 -MTJU0_SX400 -MTKD0_SI1187 -MTKD0_SI1817 -MTKD0_SI630 -MTKD0_SX107 -MTKD0_SX17 -MTKD0_SX197 -MTKD0_SX287 -MTKD0_SX377 -MTKP0_SI1023 -MTKP0_SI2283 -MTKP0_SI454 -MTKP0_SX123 -MTKP0_SX213 -MTKP0_SX303 -MTKP0_SX33 -MTKP0_SX393 -MTLB0_SI1134 -MTLB0_SI1764 -MTLB0_SI504 -MTLB0_SX144 -MTLB0_SX234 -MTLB0_SX324 -MTLB0_SX414 -MTLB0_SX54 -MTLC0_SI1313 -MTLC0_SI1477 -MTLC0_SI847 -MTLC0_SX127 -MTLC0_SX217 -MTLC0_SX307 -MTLC0_SX37 -MTLC0_SX397 -MTML0_SI1065 -MTML0_SI1695 -MTML0_SI2325 -MTML0_SX165 -MTML0_SX255 -MTML0_SX345 -MTML0_SX435 -MTML0_SX75 -MTMN0_SI1064 -MTMN0_SI2324 -MTMN0_SI582 -MTMN0_SX164 -MTMN0_SX254 -MTMN0_SX344 -MTMN0_SX434 -MTMN0_SX74 -MTMT0_SI1118 -MTMT0_SI1748 -MTMT0_SI488 -MTMT0_SX128 -MTMT0_SX218 -MTMT0_SX308 -MTMT0_SX38 -MTMT0_SX398 -MTPF0_SI1235 -MTPF0_SI1865 -MTPF0_SI605 -MTPF0_SX155 -MTPF0_SX245 -MTPF0_SX335 -MTPF0_SX425 -MTPF0_SX65 -MTPG0_SI1383 -MTPG0_SI2013 -MTPG0_SI753 -MTPG0_SX123 -MTPG0_SX213 -MTPG0_SX303 -MTPG0_SX33 -MTPG0_SX393 -MTPP0_SI1508 -MTPP0_SI2138 -MTPP0_SI878 -MTPP0_SX158 -MTPP0_SX248 -MTPP0_SX338 -MTPP0_SX428 -MTPP0_SX68 -MTPR0_SI1600 -MTPR0_SI2230 -MTPR0_SI506 -MTPR0_SX160 -MTPR0_SX250 -MTPR0_SX340 -MTPR0_SX430 -MTPR0_SX70 -MTQC0_SI1441 -MTQC0_SI2071 -MTQC0_SI480 -MTQC0_SX181 -MTQC0_SX271 -MTQC0_SX361 -MTQC0_SX451 -MTQC0_SX91 -MTRC0_SI1623 -MTRC0_SI589 -MTRC0_SI993 -MTRC0_SX170 -MTRC0_SX183 -MTRC0_SX273 -MTRC0_SX363 -MTRC0_SX93 -MTRR0_SI1548 -MTRR0_SI2178 -MTRR0_SI918 -MTRR0_SX108 -MTRR0_SX18 -MTRR0_SX198 -MTRR0_SX288 -MTRR0_SX378 -MTRT0_SI1227 -MTRT0_SI1857 -MTRT0_SI597 -MTRT0_SX147 -MTRT0_SX237 -MTRT0_SX254 -MTRT0_SX417 -MTRT0_SX57 -MTWH1_SI1512 -MTWH1_SI2142 -MTWH1_SI882 -MTWH1_SX162 -MTWH1_SX252 -MTWH1_SX342 -MTWH1_SX432 -MTWH1_SX72 -MTXS0_SI1060 -MTXS0_SI1690 -MTXS0_SI2320 -MTXS0_SX160 -MTXS0_SX250 -MTXS0_SX340 -MTXS0_SX430 -MTXS0_SX70 -MVJH0_SI1556 -MVJH0_SI2186 -MVJH0_SI926 -MVJH0_SX116 -MVJH0_SX206 -MVJH0_SX26 -MVJH0_SX296 -MVJH0_SX386 -MVLO0_SI1147 -MVLO0_SI1777 -MVLO0_SI517 -MVLO0_SX157 -MVLO0_SX247 -MVLO0_SX337 -MVLO0_SX427 -MVLO0_SX67 -MVRW0_SI1485 -MVRW0_SI2115 -MVRW0_SI855 -MVRW0_SX135 -MVRW0_SX225 -MVRW0_SX315 -MVRW0_SX405 -MVRW0_SX45 -MWAC0_SI1601 -MWAC0_SI2231 -MWAC0_SI971 -MWAC0_SX161 -MWAC0_SX251 -MWAC0_SX341 -MWAC0_SX431 -MWAC0_SX71 -MWAD0_SI1062 -MWAD0_SI1749 -MWAD0_SI2322 -MWAD0_SX162 -MWAD0_SX252 -MWAD0_SX342 -MWAD0_SX432 -MWAD0_SX72 -MWAR0_SI1045 -MWAR0_SI1675 -MWAR0_SI2305 -MWAR0_SX145 -MWAR0_SX235 -MWAR0_SX325 -MWAR0_SX415 -MWAR0_SX55 -MWCH0_SI1622 -MWCH0_SI1895 -MWCH0_SI2252 -MWCH0_SX182 -MWCH0_SX272 -MWCH0_SX362 -MWCH0_SX452 -MWCH0_SX92 -MWDK0_SI1436 -MWDK0_SI2017 -MWDK0_SI806 -MWDK0_SX176 -MWDK0_SX266 -MWDK0_SX356 -MWDK0_SX446 -MWDK0_SX86 -MWEM0_SI1320 -MWEM0_SI1393 -MWEM0_SI1950 -MWEM0_SX150 -MWEM0_SX240 -MWEM0_SX330 -MWEM0_SX420 -MWEM0_SX60 -MWGR0_SI1606 -MWGR0_SI2236 -MWGR0_SI976 -MWGR0_SX166 -MWGR0_SX256 -MWGR0_SX346 -MWGR0_SX436 -MWGR0_SX76 -MWRE0_SI1057 -MWRE0_SI1687 -MWRE0_SI2317 -MWRE0_SX157 -MWRE0_SX247 -MWRE0_SX337 -MWRE0_SX427 -MWRE0_SX67 -MWRP0_SI1443 -MWRP0_SI1525 -MWRP0_SI2073 -MWRP0_SX183 -MWRP0_SX273 -MWRP0_SX3 -MWRP0_SX363 -MWRP0_SX93 -MWSB0_SI1626 -MWSB0_SI2256 -MWSB0_SI996 -MWSB0_SX186 -MWSB0_SX276 -MWSB0_SX366 -MWSB0_SX6 -MWSB0_SX96 -MWSH0_SI1426 -MWSH0_SI2266 -MWSH0_SI796 -MWSH0_SX166 -MWSH0_SX256 -MWSH0_SX346 -MWSH0_SX436 -MWSH0_SX76 -MZMB0_SI1166 -MZMB0_SI1796 -MZMB0_SI536 -MZMB0_SX176 -MZMB0_SX266 -MZMB0_SX356 -MZMB0_SX446 -MZMB0_SX86 diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/config/timit_matched/valid.uid b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/config/timit_matched/valid.uid deleted file mode 100644 index ab5ef381ab9319aa9aefa9054e0a5128aec6f5e1..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/config/timit_matched/valid.uid +++ /dev/null @@ -1,400 +0,0 @@ -FADG0_SI1279 -FADG0_SI1909 -FADG0_SI649 -FADG0_SX109 -FADG0_SX19 -FADG0_SX199 -FADG0_SX289 -FADG0_SX379 -FAKS0_SI1573 -FAKS0_SI2203 -FAKS0_SI943 -FAKS0_SX133 -FAKS0_SX223 -FAKS0_SX313 -FAKS0_SX403 -FAKS0_SX43 -FCAL1_SI1403 -FCAL1_SI2033 -FCAL1_SI773 -FCAL1_SX143 -FCAL1_SX233 -FCAL1_SX323 -FCAL1_SX413 -FCAL1_SX53 -FCMH0_SI1454 -FCMH0_SI2084 -FCMH0_SI824 -FCMH0_SX104 -FCMH0_SX14 -FCMH0_SX194 -FCMH0_SX284 -FCMH0_SX374 -FDAC1_SI1474 -FDAC1_SI2104 -FDAC1_SI844 -FDAC1_SX124 -FDAC1_SX214 -FDAC1_SX304 -FDAC1_SX34 -FDAC1_SX394 -FDMS0_SI1218 -FDMS0_SI1502 -FDMS0_SI1848 -FDMS0_SX138 -FDMS0_SX228 -FDMS0_SX318 -FDMS0_SX408 -FDMS0_SX48 -FDRW0_SI1283 -FDRW0_SI1423 -FDRW0_SI653 -FDRW0_SX113 -FDRW0_SX203 -FDRW0_SX23 -FDRW0_SX293 -FDRW0_SX383 -FEDW0_SI1084 -FEDW0_SI1653 -FEDW0_SI1714 -FEDW0_SX184 -FEDW0_SX274 -FEDW0_SX364 -FEDW0_SX4 -FEDW0_SX94 -FGJD0_SI1179 -FGJD0_SI549 -FGJD0_SI818 -FGJD0_SX189 -FGJD0_SX279 -FGJD0_SX369 -FGJD0_SX9 -FGJD0_SX99 -FJEM0_SI1264 -FJEM0_SI1894 -FJEM0_SI634 -FJEM0_SX184 -FJEM0_SX274 -FJEM0_SX364 -FJEM0_SX4 -FJEM0_SX94 -FJMG0_SI1181 -FJMG0_SI1811 -FJMG0_SI551 -FJMG0_SX101 -FJMG0_SX11 -FJMG0_SX191 -FJMG0_SX281 -FJMG0_SX371 -FJSJ0_SI1484 -FJSJ0_SI2114 -FJSJ0_SI854 -FJSJ0_SX134 -FJSJ0_SX224 -FJSJ0_SX314 -FJSJ0_SX404 -FJSJ0_SX44 -FKMS0_SI1490 -FKMS0_SI2120 -FKMS0_SI860 -FKMS0_SX140 -FKMS0_SX230 -FKMS0_SX320 -FKMS0_SX410 -FKMS0_SX50 -FMAH0_SI1289 -FMAH0_SI1919 -FMAH0_SI659 -FMAH0_SX119 -FMAH0_SX209 -FMAH0_SX29 -FMAH0_SX299 -FMAH0_SX389 -FMML0_SI1040 -FMML0_SI1670 -FMML0_SI2300 -FMML0_SX140 -FMML0_SX230 -FMML0_SX320 -FMML0_SX410 -FMML0_SX50 -FNMR0_SI1399 -FNMR0_SI2029 -FNMR0_SI769 -FNMR0_SX139 -FNMR0_SX229 -FNMR0_SX319 -FNMR0_SX409 -FNMR0_SX49 -FREW0_SI1030 -FREW0_SI1280 -FREW0_SI1910 -FREW0_SX110 -FREW0_SX20 -FREW0_SX200 -FREW0_SX290 -FREW0_SX380 -FSEM0_SI1198 -FSEM0_SI1828 -FSEM0_SI568 -FSEM0_SX118 -FSEM0_SX208 -FSEM0_SX28 -FSEM0_SX298 -FSEM0_SX388 -MAJC0_SI1946 -MAJC0_SI2095 -MAJC0_SI835 -MAJC0_SX115 -MAJC0_SX205 -MAJC0_SX25 -MAJC0_SX295 -MAJC0_SX385 -MBDG0_SI1463 -MBDG0_SI2093 -MBDG0_SI833 -MBDG0_SX113 -MBDG0_SX203 -MBDG0_SX23 -MBDG0_SX293 -MBDG0_SX383 -MBNS0_SI1220 -MBNS0_SI1850 -MBNS0_SI590 -MBNS0_SX140 -MBNS0_SX230 -MBNS0_SX320 -MBNS0_SX410 -MBNS0_SX50 -MBWM0_SI1304 -MBWM0_SI1934 -MBWM0_SI674 -MBWM0_SX134 -MBWM0_SX224 -MBWM0_SX314 -MBWM0_SX404 -MBWM0_SX44 -MCSH0_SI1549 -MCSH0_SI2179 -MCSH0_SI919 -MCSH0_SX109 -MCSH0_SX19 -MCSH0_SX199 -MCSH0_SX289 -MCSH0_SX379 -MDLF0_SI1583 -MDLF0_SI2213 -MDLF0_SI953 -MDLF0_SX143 -MDLF0_SX233 -MDLF0_SX323 -MDLF0_SX413 -MDLF0_SX53 -MDLS0_SI1628 -MDLS0_SI2258 -MDLS0_SI998 -MDLS0_SX188 -MDLS0_SX278 -MDLS0_SX368 -MDLS0_SX8 -MDLS0_SX98 -MDVC0_SI2174 -MDVC0_SI2196 -MDVC0_SI936 -MDVC0_SX126 -MDVC0_SX216 -MDVC0_SX306 -MDVC0_SX36 -MDVC0_SX396 -MERS0_SI1019 -MERS0_SI1649 -MERS0_SI497 -MERS0_SX119 -MERS0_SX209 -MERS0_SX29 -MERS0_SX299 -MERS0_SX389 -MGJF0_SI1901 -MGJF0_SI641 -MGJF0_SI776 -MGJF0_SX101 -MGJF0_SX11 -MGJF0_SX191 -MGJF0_SX281 -MGJF0_SX371 -MGLB0_SI1534 -MGLB0_SI2164 -MGLB0_SI904 -MGLB0_SX184 -MGLB0_SX274 -MGLB0_SX364 -MGLB0_SX4 -MGLB0_SX94 -MGWT0_SI1539 -MGWT0_SI2169 -MGWT0_SI909 -MGWT0_SX189 -MGWT0_SX279 -MGWT0_SX369 -MGWT0_SX9 -MGWT0_SX99 -MJAR0_SI1988 -MJAR0_SI2247 -MJAR0_SI728 -MJAR0_SX188 -MJAR0_SX278 -MJAR0_SX368 -MJAR0_SX8 -MJAR0_SX98 -MJFC0_SI1033 -MJFC0_SI1663 -MJFC0_SI2293 -MJFC0_SX133 -MJFC0_SX223 -MJFC0_SX313 -MJFC0_SX403 -MJFC0_SX43 -MJSW0_SI1010 -MJSW0_SI1640 -MJSW0_SI2270 -MJSW0_SX110 -MJSW0_SX20 -MJSW0_SX200 -MJSW0_SX290 -MJSW0_SX380 -MMDB1_SI1625 -MMDB1_SI2255 -MMDB1_SI995 -MMDB1_SX185 -MMDB1_SX275 -MMDB1_SX365 -MMDB1_SX5 -MMDB1_SX95 -MMDM2_SI1452 -MMDM2_SI1555 -MMDM2_SI2082 -MMDM2_SX102 -MMDM2_SX12 -MMDM2_SX192 -MMDM2_SX282 -MMDM2_SX372 -MMJR0_SI1648 -MMJR0_SI2166 -MMJR0_SI2278 -MMJR0_SX118 -MMJR0_SX208 -MMJR0_SX28 -MMJR0_SX298 -MMJR0_SX388 -MMWH0_SI1089 -MMWH0_SI1301 -MMWH0_SI459 -MMWH0_SX189 -MMWH0_SX279 -MMWH0_SX369 -MMWH0_SX9 -MMWH0_SX99 -MPDF0_SI1542 -MPDF0_SI2172 -MPDF0_SI912 -MPDF0_SX102 -MPDF0_SX12 -MPDF0_SX192 -MPDF0_SX282 -MPDF0_SX372 -MRCS0_SI1223 -MRCS0_SI1853 -MRCS0_SI593 -MRCS0_SX143 -MRCS0_SX233 -MRCS0_SX323 -MRCS0_SX413 -MRCS0_SX53 -MREB0_SI1375 -MREB0_SI2005 -MREB0_SI745 -MREB0_SX115 -MREB0_SX205 -MREB0_SX25 -MREB0_SX295 -MREB0_SX385 -MRJM4_SI1489 -MRJM4_SI2119 -MRJM4_SI859 -MRJM4_SX139 -MRJM4_SX229 -MRJM4_SX319 -MRJM4_SX409 -MRJM4_SX49 -MRJR0_SI1182 -MRJR0_SI1812 -MRJR0_SI2313 -MRJR0_SX102 -MRJR0_SX12 -MRJR0_SX192 -MRJR0_SX282 -MRJR0_SX372 -MROA0_SI1307 -MROA0_SI1970 -MROA0_SI677 -MROA0_SX137 -MROA0_SX227 -MROA0_SX317 -MROA0_SX407 -MROA0_SX47 -MRTK0_SI1093 -MRTK0_SI1723 -MRTK0_SI1750 -MRTK0_SX103 -MRTK0_SX13 -MRTK0_SX193 -MRTK0_SX283 -MRTK0_SX373 -MRWS1_SI1130 -MRWS1_SI1496 -MRWS1_SI500 -MRWS1_SX140 -MRWS1_SX230 -MRWS1_SX320 -MRWS1_SX410 -MRWS1_SX50 -MTAA0_SI1285 -MTAA0_SI1915 -MTAA0_SI596 -MTAA0_SX115 -MTAA0_SX205 -MTAA0_SX25 -MTAA0_SX295 -MTAA0_SX385 -MTDT0_SI1994 -MTDT0_SI2254 -MTDT0_SI994 -MTDT0_SX184 -MTDT0_SX274 -MTDT0_SX364 -MTDT0_SX4 -MTDT0_SX94 -MTEB0_SI1133 -MTEB0_SI2064 -MTEB0_SI503 -MTEB0_SX143 -MTEB0_SX233 -MTEB0_SX323 -MTEB0_SX413 -MTEB0_SX53 -MTHC0_SI1015 -MTHC0_SI1645 -MTHC0_SI2275 -MTHC0_SX115 -MTHC0_SX205 -MTHC0_SX25 -MTHC0_SX295 -MTHC0_SX385 -MWJG0_SI1124 -MWJG0_SI1754 -MWJG0_SI494 -MWJG0_SX134 -MWJG0_SX224 -MWJG0_SX314 -MWJG0_SX404 -MWJG0_SX44 diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/config/timit_unmatched/test.uid b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/config/timit_unmatched/test.uid deleted file mode 100644 index e3967e42423d4d82d159cb395514d41c13316da2..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/config/timit_unmatched/test.uid +++ /dev/null @@ -1,1680 +0,0 @@ -FADG0_SA1 -FADG0_SA2 -FADG0_SI1279 -FADG0_SI1909 -FADG0_SI649 -FADG0_SX109 -FADG0_SX19 -FADG0_SX199 -FADG0_SX289 -FADG0_SX379 -FAKS0_SA1 -FAKS0_SA2 -FAKS0_SI1573 -FAKS0_SI2203 -FAKS0_SI943 -FAKS0_SX133 -FAKS0_SX223 -FAKS0_SX313 -FAKS0_SX403 -FAKS0_SX43 -FASW0_SA1 -FASW0_SA2 -FASW0_SI1550 -FASW0_SI2180 -FASW0_SI920 -FASW0_SX110 -FASW0_SX20 -FASW0_SX200 -FASW0_SX290 -FASW0_SX380 -FAWF0_SA1 -FAWF0_SA2 -FAWF0_SI1000 -FAWF0_SI1630 -FAWF0_SI2260 -FAWF0_SX10 -FAWF0_SX100 -FAWF0_SX190 -FAWF0_SX280 -FAWF0_SX370 -FCAL1_SA1 -FCAL1_SA2 -FCAL1_SI1403 -FCAL1_SI2033 -FCAL1_SI773 -FCAL1_SX143 -FCAL1_SX233 -FCAL1_SX323 -FCAL1_SX413 -FCAL1_SX53 -FCAU0_SA1 -FCAU0_SA2 -FCAU0_SI1037 -FCAU0_SI1667 -FCAU0_SI2297 -FCAU0_SX137 -FCAU0_SX227 -FCAU0_SX317 -FCAU0_SX407 -FCAU0_SX47 -FCFT0_SA1 -FCFT0_SA2 -FCFT0_SI1178 -FCFT0_SI1808 -FCFT0_SI548 -FCFT0_SX188 -FCFT0_SX278 -FCFT0_SX368 -FCFT0_SX8 -FCFT0_SX98 -FCMH0_SA1 -FCMH0_SA2 -FCMH0_SI1454 -FCMH0_SI2084 -FCMH0_SI824 -FCMH0_SX104 -FCMH0_SX14 -FCMH0_SX194 -FCMH0_SX284 -FCMH0_SX374 -FCMH1_SA1 -FCMH1_SA2 -FCMH1_SI1493 -FCMH1_SI2123 -FCMH1_SI863 -FCMH1_SX143 -FCMH1_SX233 -FCMH1_SX323 -FCMH1_SX413 -FCMH1_SX53 -FCMR0_SA1 -FCMR0_SA2 -FCMR0_SI1105 -FCMR0_SI1735 -FCMR0_SI475 -FCMR0_SX115 -FCMR0_SX205 -FCMR0_SX25 -FCMR0_SX295 -FCMR0_SX385 -FCRH0_SA1 -FCRH0_SA2 -FCRH0_SI1088 -FCRH0_SI1718 -FCRH0_SI458 -FCRH0_SX188 -FCRH0_SX278 -FCRH0_SX368 -FCRH0_SX8 -FCRH0_SX98 -FDAC1_SA1 -FDAC1_SA2 -FDAC1_SI1474 -FDAC1_SI2104 -FDAC1_SI844 -FDAC1_SX124 -FDAC1_SX214 -FDAC1_SX304 -FDAC1_SX34 -FDAC1_SX394 -FDHC0_SA1 -FDHC0_SA2 -FDHC0_SI1559 -FDHC0_SI2189 -FDHC0_SI929 -FDHC0_SX119 -FDHC0_SX209 -FDHC0_SX29 -FDHC0_SX299 -FDHC0_SX389 -FDMS0_SA1 -FDMS0_SA2 -FDMS0_SI1218 -FDMS0_SI1502 -FDMS0_SI1848 -FDMS0_SX138 -FDMS0_SX228 -FDMS0_SX318 -FDMS0_SX408 -FDMS0_SX48 -FDRD1_SA1 -FDRD1_SA2 -FDRD1_SI1544 -FDRD1_SI1566 -FDRD1_SI2149 -FDRD1_SX104 -FDRD1_SX14 -FDRD1_SX194 -FDRD1_SX284 -FDRD1_SX374 -FDRW0_SA1 -FDRW0_SA2 -FDRW0_SI1283 -FDRW0_SI1423 -FDRW0_SI653 -FDRW0_SX113 -FDRW0_SX203 -FDRW0_SX23 -FDRW0_SX293 -FDRW0_SX383 -FEDW0_SA1 -FEDW0_SA2 -FEDW0_SI1084 -FEDW0_SI1653 -FEDW0_SI1714 -FEDW0_SX184 -FEDW0_SX274 -FEDW0_SX364 -FEDW0_SX4 -FEDW0_SX94 -FELC0_SA1 -FELC0_SA2 -FELC0_SI1386 -FELC0_SI2016 -FELC0_SI756 -FELC0_SX126 -FELC0_SX216 -FELC0_SX306 -FELC0_SX36 -FELC0_SX396 -FGJD0_SA1 -FGJD0_SA2 -FGJD0_SI1179 -FGJD0_SI549 -FGJD0_SI818 -FGJD0_SX189 -FGJD0_SX279 -FGJD0_SX369 -FGJD0_SX9 -FGJD0_SX99 -FGMD0_SA1 -FGMD0_SA2 -FGMD0_SI1943 -FGMD0_SI2107 -FGMD0_SI683 -FGMD0_SX143 -FGMD0_SX233 -FGMD0_SX323 -FGMD0_SX413 -FGMD0_SX53 -FGWR0_SA1 -FGWR0_SA2 -FGWR0_SI1578 -FGWR0_SI2208 -FGWR0_SI948 -FGWR0_SX138 -FGWR0_SX228 -FGWR0_SX318 -FGWR0_SX408 -FGWR0_SX48 -FHES0_SA1 -FHES0_SA2 -FHES0_SI1109 -FHES0_SI1739 -FHES0_SI479 -FHES0_SX119 -FHES0_SX209 -FHES0_SX29 -FHES0_SX299 -FHES0_SX389 -FHEW0_SA1 -FHEW0_SA2 -FHEW0_SI2023 -FHEW0_SI690 -FHEW0_SI763 -FHEW0_SX133 -FHEW0_SX223 -FHEW0_SX313 -FHEW0_SX403 -FHEW0_SX43 -FISB0_SA1 -FISB0_SA2 -FISB0_SI1579 -FISB0_SI2209 -FISB0_SI949 -FISB0_SX139 -FISB0_SX229 -FISB0_SX319 -FISB0_SX409 -FISB0_SX49 -FJAS0_SA1 -FJAS0_SA2 -FJAS0_SI1400 -FJAS0_SI2030 -FJAS0_SI770 -FJAS0_SX140 -FJAS0_SX230 -FJAS0_SX320 -FJAS0_SX410 -FJAS0_SX50 -FJCS0_SA1 -FJCS0_SA2 -FJCS0_SI1309 -FJCS0_SI1833 -FJCS0_SI1939 -FJCS0_SX139 -FJCS0_SX229 -FJCS0_SX319 -FJCS0_SX409 -FJCS0_SX49 -FJEM0_SA1 -FJEM0_SA2 -FJEM0_SI1264 -FJEM0_SI1894 -FJEM0_SI634 -FJEM0_SX184 -FJEM0_SX274 -FJEM0_SX364 -FJEM0_SX4 -FJEM0_SX94 -FJLM0_SA1 -FJLM0_SA2 -FJLM0_SI1043 -FJLM0_SI1673 -FJLM0_SI2303 -FJLM0_SX143 -FJLM0_SX233 -FJLM0_SX323 -FJLM0_SX413 -FJLM0_SX53 -FJMG0_SA1 -FJMG0_SA2 -FJMG0_SI1181 -FJMG0_SI1811 -FJMG0_SI551 -FJMG0_SX101 -FJMG0_SX11 -FJMG0_SX191 -FJMG0_SX281 -FJMG0_SX371 -FJRE0_SA1 -FJRE0_SA2 -FJRE0_SI1116 -FJRE0_SI1587 -FJRE0_SI1746 -FJRE0_SX126 -FJRE0_SX216 -FJRE0_SX306 -FJRE0_SX36 -FJRE0_SX396 -FJSA0_SA1 -FJSA0_SA2 -FJSA0_SI1379 -FJSA0_SI2009 -FJSA0_SI749 -FJSA0_SX119 -FJSA0_SX209 -FJSA0_SX29 -FJSA0_SX299 -FJSA0_SX389 -FJSJ0_SA1 -FJSJ0_SA2 -FJSJ0_SI1484 -FJSJ0_SI2114 -FJSJ0_SI854 -FJSJ0_SX134 -FJSJ0_SX224 -FJSJ0_SX314 -FJSJ0_SX404 -FJSJ0_SX44 -FJWB0_SA1 -FJWB0_SA2 -FJWB0_SI1265 -FJWB0_SI635 -FJWB0_SI992 -FJWB0_SX185 -FJWB0_SX275 -FJWB0_SX365 -FJWB0_SX5 -FJWB0_SX95 -FKMS0_SA1 -FKMS0_SA2 -FKMS0_SI1490 -FKMS0_SI2120 -FKMS0_SI860 -FKMS0_SX140 -FKMS0_SX230 -FKMS0_SX320 -FKMS0_SX410 -FKMS0_SX50 -FLAS0_SA1 -FLAS0_SA2 -FLAS0_SI1026 -FLAS0_SI1488 -FLAS0_SI858 -FLAS0_SX138 -FLAS0_SX228 -FLAS0_SX318 -FLAS0_SX408 -FLAS0_SX48 -FLBW0_SA1 -FLBW0_SA2 -FLBW0_SI1219 -FLBW0_SI1849 -FLBW0_SI2253 -FLBW0_SX139 -FLBW0_SX229 -FLBW0_SX319 -FLBW0_SX409 -FLBW0_SX49 -FLKD0_SA1 -FLKD0_SA2 -FLKD0_SI1369 -FLKD0_SI739 -FLKD0_SI894 -FLKD0_SX109 -FLKD0_SX19 -FLKD0_SX199 -FLKD0_SX289 -FLKD0_SX379 -FLNH0_SA1 -FLNH0_SA2 -FLNH0_SI1214 -FLNH0_SI584 -FLNH0_SI941 -FLNH0_SX134 -FLNH0_SX224 -FLNH0_SX314 -FLNH0_SX404 -FLNH0_SX44 -FMAF0_SA1 -FMAF0_SA2 -FMAF0_SI1459 -FMAF0_SI2089 -FMAF0_SI829 -FMAF0_SX109 -FMAF0_SX19 -FMAF0_SX199 -FMAF0_SX289 -FMAF0_SX379 -FMAH0_SA1 -FMAH0_SA2 -FMAH0_SI1289 -FMAH0_SI1919 -FMAH0_SI659 -FMAH0_SX119 -FMAH0_SX209 -FMAH0_SX29 -FMAH0_SX299 -FMAH0_SX389 -FMCM0_SA1 -FMCM0_SA2 -FMCM0_SI1180 -FMCM0_SI1810 -FMCM0_SI550 -FMCM0_SX10 -FMCM0_SX100 -FMCM0_SX190 -FMCM0_SX280 -FMCM0_SX370 -FMGD0_SA1 -FMGD0_SA2 -FMGD0_SI1564 -FMGD0_SI2194 -FMGD0_SI934 -FMGD0_SX124 -FMGD0_SX214 -FMGD0_SX304 -FMGD0_SX34 -FMGD0_SX394 -FMLD0_SA1 -FMLD0_SA2 -FMLD0_SI2185 -FMLD0_SI822 -FMLD0_SI925 -FMLD0_SX115 -FMLD0_SX205 -FMLD0_SX25 -FMLD0_SX295 -FMLD0_SX385 -FMML0_SA1 -FMML0_SA2 -FMML0_SI1040 -FMML0_SI1670 -FMML0_SI2300 -FMML0_SX140 -FMML0_SX230 -FMML0_SX320 -FMML0_SX410 -FMML0_SX50 -FNLP0_SA1 -FNLP0_SA2 -FNLP0_SI1308 -FNLP0_SI1938 -FNLP0_SI678 -FNLP0_SX138 -FNLP0_SX228 -FNLP0_SX318 -FNLP0_SX408 -FNLP0_SX48 -FNMR0_SA1 -FNMR0_SA2 -FNMR0_SI1399 -FNMR0_SI2029 -FNMR0_SI769 -FNMR0_SX139 -FNMR0_SX229 -FNMR0_SX319 -FNMR0_SX409 -FNMR0_SX49 -FPAS0_SA1 -FPAS0_SA2 -FPAS0_SI1272 -FPAS0_SI2204 -FPAS0_SI944 -FPAS0_SX134 -FPAS0_SX224 -FPAS0_SX314 -FPAS0_SX404 -FPAS0_SX44 -FPKT0_SA1 -FPKT0_SA2 -FPKT0_SI1538 -FPKT0_SI2168 -FPKT0_SI908 -FPKT0_SX188 -FPKT0_SX278 -FPKT0_SX368 -FPKT0_SX8 -FPKT0_SX98 -FRAM1_SA1 -FRAM1_SA2 -FRAM1_SI1360 -FRAM1_SI522 -FRAM1_SI730 -FRAM1_SX10 -FRAM1_SX100 -FRAM1_SX190 -FRAM1_SX280 -FRAM1_SX370 -FREW0_SA1 -FREW0_SA2 -FREW0_SI1030 -FREW0_SI1280 -FREW0_SI1910 -FREW0_SX110 -FREW0_SX20 -FREW0_SX200 -FREW0_SX290 -FREW0_SX380 -FRNG0_SA1 -FRNG0_SA2 -FRNG0_SI1355 -FRNG0_SI1985 -FRNG0_SI725 -FRNG0_SX185 -FRNG0_SX275 -FRNG0_SX365 -FRNG0_SX5 -FRNG0_SX95 -FSEM0_SA1 -FSEM0_SA2 -FSEM0_SI1198 -FSEM0_SI1828 -FSEM0_SI568 -FSEM0_SX118 -FSEM0_SX208 -FSEM0_SX28 -FSEM0_SX298 -FSEM0_SX388 -FSLB1_SA1 -FSLB1_SA2 -FSLB1_SI1904 -FSLB1_SI644 -FSLB1_SI891 -FSLB1_SX104 -FSLB1_SX14 -FSLB1_SX194 -FSLB1_SX284 -FSLB1_SX374 -FSXA0_SA1 -FSXA0_SA2 -FSXA0_SI1108 -FSXA0_SI1846 -FSXA0_SI478 -FSXA0_SX118 -FSXA0_SX208 -FSXA0_SX28 -FSXA0_SX298 -FSXA0_SX388 -FTLH0_SA1 -FTLH0_SA2 -FTLH0_SI1009 -FTLH0_SI1390 -FTLH0_SI1639 -FTLH0_SX109 -FTLH0_SX19 -FTLH0_SX199 -FTLH0_SX289 -FTLH0_SX379 -FUTB0_SA1 -FUTB0_SA2 -FUTB0_SI1204 -FUTB0_SI1330 -FUTB0_SI1834 -FUTB0_SX124 -FUTB0_SX214 -FUTB0_SX304 -FUTB0_SX34 -FUTB0_SX394 -MABW0_SA1 -MABW0_SA2 -MABW0_SI1230 -MABW0_SI1664 -MABW0_SI2294 -MABW0_SX134 -MABW0_SX224 -MABW0_SX314 -MABW0_SX404 -MABW0_SX44 -MAHH0_SA1 -MAHH0_SA2 -MAHH0_SI1294 -MAHH0_SI1924 -MAHH0_SI664 -MAHH0_SX124 -MAHH0_SX214 -MAHH0_SX304 -MAHH0_SX34 -MAHH0_SX394 -MAJC0_SA1 -MAJC0_SA2 -MAJC0_SI1946 -MAJC0_SI2095 -MAJC0_SI835 -MAJC0_SX115 -MAJC0_SX205 -MAJC0_SX25 -MAJC0_SX295 -MAJC0_SX385 -MBDG0_SA1 -MBDG0_SA2 -MBDG0_SI1463 -MBDG0_SI2093 -MBDG0_SI833 -MBDG0_SX113 -MBDG0_SX203 -MBDG0_SX23 -MBDG0_SX293 -MBDG0_SX383 -MBJK0_SA1 -MBJK0_SA2 -MBJK0_SI1175 -MBJK0_SI2128 -MBJK0_SI545 -MBJK0_SX185 -MBJK0_SX275 -MBJK0_SX365 -MBJK0_SX5 -MBJK0_SX95 -MBNS0_SA1 -MBNS0_SA2 -MBNS0_SI1220 -MBNS0_SI1850 -MBNS0_SI590 -MBNS0_SX140 -MBNS0_SX230 -MBNS0_SX320 -MBNS0_SX410 -MBNS0_SX50 -MBPM0_SA1 -MBPM0_SA2 -MBPM0_SI1577 -MBPM0_SI1584 -MBPM0_SI947 -MBPM0_SX137 -MBPM0_SX227 -MBPM0_SX317 -MBPM0_SX407 -MBPM0_SX47 -MBWM0_SA1 -MBWM0_SA2 -MBWM0_SI1304 -MBWM0_SI1934 -MBWM0_SI674 -MBWM0_SX134 -MBWM0_SX224 -MBWM0_SX314 -MBWM0_SX404 -MBWM0_SX44 -MCCS0_SA1 -MCCS0_SA2 -MCCS0_SI1469 -MCCS0_SI2099 -MCCS0_SI839 -MCCS0_SX119 -MCCS0_SX209 -MCCS0_SX29 -MCCS0_SX299 -MCCS0_SX389 -MCEM0_SA1 -MCEM0_SA2 -MCEM0_SI1398 -MCEM0_SI2028 -MCEM0_SI768 -MCEM0_SX138 -MCEM0_SX228 -MCEM0_SX318 -MCEM0_SX408 -MCEM0_SX48 -MCHH0_SA1 -MCHH0_SA2 -MCHH0_SI1004 -MCHH0_SI1634 -MCHH0_SI530 -MCHH0_SX104 -MCHH0_SX14 -MCHH0_SX194 -MCHH0_SX284 -MCHH0_SX374 -MCMB0_SA1 -MCMB0_SA2 -MCMB0_SI1268 -MCMB0_SI1898 -MCMB0_SI638 -MCMB0_SX188 -MCMB0_SX278 -MCMB0_SX368 -MCMB0_SX8 -MCMB0_SX98 -MCMJ0_SA1 -MCMJ0_SA2 -MCMJ0_SI1094 -MCMJ0_SI464 -MCMJ0_SI602 -MCMJ0_SX104 -MCMJ0_SX14 -MCMJ0_SX194 -MCMJ0_SX284 -MCMJ0_SX374 -MCRC0_SA1 -MCRC0_SA2 -MCRC0_SI1092 -MCRC0_SI1722 -MCRC0_SI462 -MCRC0_SX102 -MCRC0_SX12 -MCRC0_SX192 -MCRC0_SX282 -MCRC0_SX372 -MCSH0_SA1 -MCSH0_SA2 -MCSH0_SI1549 -MCSH0_SI2179 -MCSH0_SI919 -MCSH0_SX109 -MCSH0_SX19 -MCSH0_SX199 -MCSH0_SX289 -MCSH0_SX379 -MCTT0_SA1 -MCTT0_SA2 -MCTT0_SI1144 -MCTT0_SI2188 -MCTT0_SI928 -MCTT0_SX118 -MCTT0_SX208 -MCTT0_SX28 -MCTT0_SX298 -MCTT0_SX388 -MCTW0_SA1 -MCTW0_SA2 -MCTW0_SI1373 -MCTW0_SI2003 -MCTW0_SI743 -MCTW0_SX113 -MCTW0_SX203 -MCTW0_SX23 -MCTW0_SX293 -MCTW0_SX383 -MDAB0_SA1 -MDAB0_SA2 -MDAB0_SI1039 -MDAB0_SI1669 -MDAB0_SI2299 -MDAB0_SX139 -MDAB0_SX229 -MDAB0_SX319 -MDAB0_SX409 -MDAB0_SX49 -MDAC2_SA1 -MDAC2_SA2 -MDAC2_SI2259 -MDAC2_SI560 -MDAC2_SI999 -MDAC2_SX189 -MDAC2_SX279 -MDAC2_SX369 -MDAC2_SX9 -MDAC2_SX99 -MDAW1_SA1 -MDAW1_SA2 -MDAW1_SI1453 -MDAW1_SI2083 -MDAW1_SI823 -MDAW1_SX103 -MDAW1_SX13 -MDAW1_SX193 -MDAW1_SX283 -MDAW1_SX373 -MDBB0_SA1 -MDBB0_SA2 -MDBB0_SI1195 -MDBB0_SI1825 -MDBB0_SI565 -MDBB0_SX115 -MDBB0_SX205 -MDBB0_SX25 -MDBB0_SX295 -MDBB0_SX385 -MDLD0_SA1 -MDLD0_SA2 -MDLD0_SI1543 -MDLD0_SI2173 -MDLD0_SI913 -MDLD0_SX103 -MDLD0_SX13 -MDLD0_SX193 -MDLD0_SX283 -MDLD0_SX373 -MDLF0_SA1 -MDLF0_SA2 -MDLF0_SI1583 -MDLF0_SI2213 -MDLF0_SI953 -MDLF0_SX143 -MDLF0_SX233 -MDLF0_SX323 -MDLF0_SX413 -MDLF0_SX53 -MDLS0_SA1 -MDLS0_SA2 -MDLS0_SI1628 -MDLS0_SI2258 -MDLS0_SI998 -MDLS0_SX188 -MDLS0_SX278 -MDLS0_SX368 -MDLS0_SX8 -MDLS0_SX98 -MDRB0_SA1 -MDRB0_SA2 -MDRB0_SI1174 -MDRB0_SI2109 -MDRB0_SI544 -MDRB0_SX184 -MDRB0_SX274 -MDRB0_SX364 -MDRB0_SX4 -MDRB0_SX94 -MDRM0_SA1 -MDRM0_SA2 -MDRM0_SI1013 -MDRM0_SI1643 -MDRM0_SI2273 -MDRM0_SX113 -MDRM0_SX203 -MDRM0_SX23 -MDRM0_SX293 -MDRM0_SX383 -MDSC0_SA1 -MDSC0_SA2 -MDSC0_SI1038 -MDSC0_SI2298 -MDSC0_SI967 -MDSC0_SX138 -MDSC0_SX228 -MDSC0_SX318 -MDSC0_SX408 -MDSC0_SX48 -MDVC0_SA1 -MDVC0_SA2 -MDVC0_SI2174 -MDVC0_SI2196 -MDVC0_SI936 -MDVC0_SX126 -MDVC0_SX216 -MDVC0_SX306 -MDVC0_SX36 -MDVC0_SX396 -MDWA0_SA1 -MDWA0_SA2 -MDWA0_SI1146 -MDWA0_SI1445 -MDWA0_SI519 -MDWA0_SX185 -MDWA0_SX275 -MDWA0_SX365 -MDWA0_SX5 -MDWA0_SX95 -MDWK0_SA1 -MDWK0_SA2 -MDWK0_SI1540 -MDWK0_SI2170 -MDWK0_SI910 -MDWK0_SX10 -MDWK0_SX100 -MDWK0_SX190 -MDWK0_SX280 -MDWK0_SX370 -MERS0_SA1 -MERS0_SA2 -MERS0_SI1019 -MERS0_SI1649 -MERS0_SI497 -MERS0_SX119 -MERS0_SX209 -MERS0_SX29 -MERS0_SX299 -MERS0_SX389 -MESD0_SA1 -MESD0_SA2 -MESD0_SI1002 -MESD0_SI1632 -MESD0_SI2262 -MESD0_SX102 -MESD0_SX12 -MESD0_SX192 -MESD0_SX282 -MESD0_SX372 -MFGK0_SA1 -MFGK0_SA2 -MFGK0_SI1451 -MFGK0_SI1744 -MFGK0_SI484 -MFGK0_SX124 -MFGK0_SX214 -MFGK0_SX304 -MFGK0_SX34 -MFGK0_SX394 -MGJF0_SA1 -MGJF0_SA2 -MGJF0_SI1901 -MGJF0_SI641 -MGJF0_SI776 -MGJF0_SX101 -MGJF0_SX11 -MGJF0_SX191 -MGJF0_SX281 -MGJF0_SX371 -MGLB0_SA1 -MGLB0_SA2 -MGLB0_SI1534 -MGLB0_SI2164 -MGLB0_SI904 -MGLB0_SX184 -MGLB0_SX274 -MGLB0_SX364 -MGLB0_SX4 -MGLB0_SX94 -MGMM0_SA1 -MGMM0_SA2 -MGMM0_SI1129 -MGMM0_SI1759 -MGMM0_SI499 -MGMM0_SX139 -MGMM0_SX229 -MGMM0_SX319 -MGMM0_SX409 -MGMM0_SX49 -MGRT0_SA1 -MGRT0_SA2 -MGRT0_SI1450 -MGRT0_SI2080 -MGRT0_SI820 -MGRT0_SX10 -MGRT0_SX100 -MGRT0_SX190 -MGRT0_SX280 -MGRT0_SX370 -MGWT0_SA1 -MGWT0_SA2 -MGWT0_SI1539 -MGWT0_SI2169 -MGWT0_SI909 -MGWT0_SX189 -MGWT0_SX279 -MGWT0_SX369 -MGWT0_SX9 -MGWT0_SX99 -MHPG0_SA1 -MHPG0_SA2 -MHPG0_SI1090 -MHPG0_SI1720 -MHPG0_SI460 -MHPG0_SX10 -MHPG0_SX100 -MHPG0_SX190 -MHPG0_SX280 -MHPG0_SX370 -MJAR0_SA1 -MJAR0_SA2 -MJAR0_SI1988 -MJAR0_SI2247 -MJAR0_SI728 -MJAR0_SX188 -MJAR0_SX278 -MJAR0_SX368 -MJAR0_SX8 -MJAR0_SX98 -MJBR0_SA1 -MJBR0_SA2 -MJBR0_SI1001 -MJBR0_SI1631 -MJBR0_SI2261 -MJBR0_SX101 -MJBR0_SX11 -MJBR0_SX191 -MJBR0_SX281 -MJBR0_SX371 -MJDH0_SA1 -MJDH0_SA2 -MJDH0_SI1354 -MJDH0_SI1984 -MJDH0_SI724 -MJDH0_SX184 -MJDH0_SX274 -MJDH0_SX364 -MJDH0_SX4 -MJDH0_SX94 -MJDM1_SA1 -MJDM1_SA2 -MJDM1_SI1085 -MJDM1_SI1715 -MJDM1_SI455 -MJDM1_SX185 -MJDM1_SX275 -MJDM1_SX365 -MJDM1_SX5 -MJDM1_SX95 -MJES0_SA1 -MJES0_SA2 -MJES0_SI1384 -MJES0_SI2014 -MJES0_SI754 -MJES0_SX124 -MJES0_SX214 -MJES0_SX304 -MJES0_SX34 -MJES0_SX394 -MJFC0_SA1 -MJFC0_SA2 -MJFC0_SI1033 -MJFC0_SI1663 -MJFC0_SI2293 -MJFC0_SX133 -MJFC0_SX223 -MJFC0_SX313 -MJFC0_SX403 -MJFC0_SX43 -MJJG0_SA1 -MJJG0_SA2 -MJJG0_SI1003 -MJJG0_SI1633 -MJJG0_SI2263 -MJJG0_SX103 -MJJG0_SX13 -MJJG0_SX193 -MJJG0_SX283 -MJJG0_SX373 -MJLN0_SA1 -MJLN0_SA2 -MJLN0_SI1449 -MJLN0_SI2079 -MJLN0_SI819 -MJLN0_SX189 -MJLN0_SX279 -MJLN0_SX369 -MJLN0_SX9 -MJLN0_SX99 -MJMP0_SA1 -MJMP0_SA2 -MJMP0_SI1535 -MJMP0_SI1791 -MJMP0_SI905 -MJMP0_SX185 -MJMP0_SX275 -MJMP0_SX365 -MJMP0_SX5 -MJMP0_SX95 -MJRF0_SA1 -MJRF0_SA2 -MJRF0_SI1114 -MJRF0_SI2081 -MJRF0_SI821 -MJRF0_SX101 -MJRF0_SX11 -MJRF0_SX191 -MJRF0_SX281 -MJRF0_SX371 -MJSW0_SA1 -MJSW0_SA2 -MJSW0_SI1010 -MJSW0_SI1640 -MJSW0_SI2270 -MJSW0_SX110 -MJSW0_SX20 -MJSW0_SX200 -MJSW0_SX290 -MJSW0_SX380 -MJTC0_SA1 -MJTC0_SA2 -MJTC0_SI1460 -MJTC0_SI2090 -MJTC0_SI830 -MJTC0_SX110 -MJTC0_SX20 -MJTC0_SX200 -MJTC0_SX290 -MJTC0_SX380 -MJTH0_SA1 -MJTH0_SA2 -MJTH0_SI1296 -MJTH0_SI1926 -MJTH0_SI666 -MJTH0_SX126 -MJTH0_SX216 -MJTH0_SX306 -MJTH0_SX36 -MJTH0_SX396 -MJVW0_SA1 -MJVW0_SA2 -MJVW0_SI1733 -MJVW0_SI1758 -MJVW0_SI473 -MJVW0_SX113 -MJVW0_SX203 -MJVW0_SX23 -MJVW0_SX293 -MJVW0_SX383 -MKCH0_SA1 -MKCH0_SA2 -MKCH0_SI1378 -MKCH0_SI1425 -MKCH0_SI2008 -MKCH0_SX118 -MKCH0_SX208 -MKCH0_SX28 -MKCH0_SX298 -MKCH0_SX388 -MKCL0_SA1 -MKCL0_SA2 -MKCL0_SI1091 -MKCL0_SI1721 -MKCL0_SI461 -MKCL0_SX101 -MKCL0_SX11 -MKCL0_SX191 -MKCL0_SX281 -MKCL0_SX371 -MKDR0_SA1 -MKDR0_SA2 -MKDR0_SI1273 -MKDR0_SI1903 -MKDR0_SI643 -MKDR0_SX103 -MKDR0_SX13 -MKDR0_SX193 -MKDR0_SX283 -MKDR0_SX373 -MKJL0_SA1 -MKJL0_SA2 -MKJL0_SI1100 -MKJL0_SI1730 -MKJL0_SI470 -MKJL0_SX110 -MKJL0_SX20 -MKJL0_SX200 -MKJL0_SX290 -MKJL0_SX380 -MKLT0_SA1 -MKLT0_SA2 -MKLT0_SI1213 -MKLT0_SI1843 -MKLT0_SI583 -MKLT0_SX133 -MKLT0_SX223 -MKLT0_SX313 -MKLT0_SX403 -MKLT0_SX43 -MLIH0_SA1 -MLIH0_SA2 -MLIH0_SI1183 -MLIH0_SI1813 -MLIH0_SI553 -MLIH0_SX103 -MLIH0_SX13 -MLIH0_SX193 -MLIH0_SX283 -MLIH0_SX373 -MLJB0_SA1 -MLJB0_SA2 -MLJB0_SI1310 -MLJB0_SI1940 -MLJB0_SI680 -MLJB0_SX140 -MLJB0_SX230 -MLJB0_SX320 -MLJB0_SX410 -MLJB0_SX50 -MLLL0_SA1 -MLLL0_SA2 -MLLL0_SI1363 -MLLL0_SI1993 -MLLL0_SI733 -MLLL0_SX103 -MLLL0_SX13 -MLLL0_SX193 -MLLL0_SX283 -MLLL0_SX373 -MLNT0_SA1 -MLNT0_SA2 -MLNT0_SI1574 -MLNT0_SI1902 -MLNT0_SI642 -MLNT0_SX102 -MLNT0_SX12 -MLNT0_SX192 -MLNT0_SX282 -MLNT0_SX372 -MMAB0_SA1 -MMAB0_SA2 -MMAB0_SI1362 -MMAB0_SI1992 -MMAB0_SI732 -MMAB0_SX102 -MMAB0_SX12 -MMAB0_SX192 -MMAB0_SX282 -MMAB0_SX372 -MMDB1_SA1 -MMDB1_SA2 -MMDB1_SI1625 -MMDB1_SI2255 -MMDB1_SI995 -MMDB1_SX185 -MMDB1_SX275 -MMDB1_SX365 -MMDB1_SX5 -MMDB1_SX95 -MMDH0_SA1 -MMDH0_SA2 -MMDH0_SI1656 -MMDH0_SI2118 -MMDH0_SI2286 -MMDH0_SX126 -MMDH0_SX216 -MMDH0_SX306 -MMDH0_SX36 -MMDH0_SX396 -MMDM2_SA1 -MMDM2_SA2 -MMDM2_SI1452 -MMDM2_SI1555 -MMDM2_SI2082 -MMDM2_SX102 -MMDM2_SX12 -MMDM2_SX192 -MMDM2_SX282 -MMDM2_SX372 -MMJR0_SA1 -MMJR0_SA2 -MMJR0_SI1648 -MMJR0_SI2166 -MMJR0_SI2278 -MMJR0_SX118 -MMJR0_SX208 -MMJR0_SX28 -MMJR0_SX298 -MMJR0_SX388 -MMWH0_SA1 -MMWH0_SA2 -MMWH0_SI1089 -MMWH0_SI1301 -MMWH0_SI459 -MMWH0_SX189 -MMWH0_SX279 -MMWH0_SX369 -MMWH0_SX9 -MMWH0_SX99 -MNJM0_SA1 -MNJM0_SA2 -MNJM0_SI1580 -MNJM0_SI2210 -MNJM0_SI950 -MNJM0_SX140 -MNJM0_SX230 -MNJM0_SX320 -MNJM0_SX410 -MNJM0_SX50 -MNLS0_SA1 -MNLS0_SA2 -MNLS0_SI1483 -MNLS0_SI1610 -MNLS0_SI853 -MNLS0_SX133 -MNLS0_SX223 -MNLS0_SX313 -MNLS0_SX403 -MNLS0_SX43 -MPAB0_SA1 -MPAB0_SA2 -MPAB0_SI1103 -MPAB0_SI1128 -MPAB0_SI498 -MPAB0_SX138 -MPAB0_SX228 -MPAB0_SX318 -MPAB0_SX408 -MPAB0_SX48 -MPAM0_SA1 -MPAM0_SA2 -MPAM0_SI1189 -MPAM0_SI1819 -MPAM0_SI1961 -MPAM0_SX109 -MPAM0_SX19 -MPAM0_SX199 -MPAM0_SX289 -MPAM0_SX379 -MPAM1_SA1 -MPAM1_SA2 -MPAM1_SI1029 -MPAM1_SI1836 -MPAM1_SI576 -MPAM1_SX126 -MPAM1_SX216 -MPAM1_SX306 -MPAM1_SX36 -MPAM1_SX396 -MPCS0_SA1 -MPCS0_SA2 -MPCS0_SI1359 -MPCS0_SI1989 -MPCS0_SI729 -MPCS0_SX189 -MPCS0_SX279 -MPCS0_SX369 -MPCS0_SX9 -MPCS0_SX99 -MPDF0_SA1 -MPDF0_SA2 -MPDF0_SI1542 -MPDF0_SI2172 -MPDF0_SI912 -MPDF0_SX102 -MPDF0_SX12 -MPDF0_SX192 -MPDF0_SX282 -MPDF0_SX372 -MPGL0_SA1 -MPGL0_SA2 -MPGL0_SI1099 -MPGL0_SI1729 -MPGL0_SI469 -MPGL0_SX109 -MPGL0_SX19 -MPGL0_SX199 -MPGL0_SX289 -MPGL0_SX379 -MPLB0_SA1 -MPLB0_SA2 -MPLB0_SI1394 -MPLB0_SI2024 -MPLB0_SI764 -MPLB0_SX134 -MPLB0_SX224 -MPLB0_SX314 -MPLB0_SX404 -MPLB0_SX44 -MPWM0_SA1 -MPWM0_SA2 -MPWM0_SI1127 -MPWM0_SI1757 -MPWM0_SI2279 -MPWM0_SX137 -MPWM0_SX227 -MPWM0_SX317 -MPWM0_SX407 -MPWM0_SX47 -MRCS0_SA1 -MRCS0_SA2 -MRCS0_SI1223 -MRCS0_SI1853 -MRCS0_SI593 -MRCS0_SX143 -MRCS0_SX233 -MRCS0_SX323 -MRCS0_SX413 -MRCS0_SX53 -MRCZ0_SA1 -MRCZ0_SA2 -MRCZ0_SI1541 -MRCZ0_SI2171 -MRCZ0_SI911 -MRCZ0_SX101 -MRCZ0_SX11 -MRCZ0_SX191 -MRCZ0_SX281 -MRCZ0_SX371 -MREB0_SA1 -MREB0_SA2 -MREB0_SI1375 -MREB0_SI2005 -MREB0_SI745 -MREB0_SX115 -MREB0_SX205 -MREB0_SX25 -MREB0_SX295 -MREB0_SX385 -MRES0_SA1 -MRES0_SA2 -MRES0_SI1217 -MRES0_SI1847 -MRES0_SI587 -MRES0_SX137 -MRES0_SX227 -MRES0_SX317 -MRES0_SX407 -MRES0_SX47 -MRGG0_SA1 -MRGG0_SA2 -MRGG0_SI1199 -MRGG0_SI1829 -MRGG0_SI569 -MRGG0_SX119 -MRGG0_SX209 -MRGG0_SX29 -MRGG0_SX299 -MRGG0_SX389 -MRJM3_SA1 -MRJM3_SA2 -MRJM3_SI1448 -MRJM3_SI1809 -MRJM3_SI2078 -MRJM3_SX188 -MRJM3_SX278 -MRJM3_SX368 -MRJM3_SX8 -MRJM3_SX98 -MRJM4_SA1 -MRJM4_SA2 -MRJM4_SI1489 -MRJM4_SI2119 -MRJM4_SI859 -MRJM4_SX139 -MRJM4_SX229 -MRJM4_SX319 -MRJM4_SX409 -MRJM4_SX49 -MRJO0_SA1 -MRJO0_SA2 -MRJO0_SI1364 -MRJO0_SI1624 -MRJO0_SI734 -MRJO0_SX104 -MRJO0_SX14 -MRJO0_SX194 -MRJO0_SX284 -MRJO0_SX374 -MRJR0_SA1 -MRJR0_SA2 -MRJR0_SI1182 -MRJR0_SI1812 -MRJR0_SI2313 -MRJR0_SX102 -MRJR0_SX12 -MRJR0_SX192 -MRJR0_SX282 -MRJR0_SX372 -MRJS0_SA1 -MRJS0_SA2 -MRJS0_SI1444 -MRJS0_SI1523 -MRJS0_SI2074 -MRJS0_SX184 -MRJS0_SX274 -MRJS0_SX364 -MRJS0_SX4 -MRJS0_SX94 -MRKO0_SA1 -MRKO0_SA2 -MRKO0_SI1397 -MRKO0_SI2027 -MRKO0_SI767 -MRKO0_SX137 -MRKO0_SX227 -MRKO0_SX317 -MRKO0_SX407 -MRKO0_SX47 -MRMS1_SA1 -MRMS1_SA2 -MRMS1_SI1487 -MRMS1_SI2117 -MRMS1_SI857 -MRMS1_SX137 -MRMS1_SX227 -MRMS1_SX317 -MRMS1_SX407 -MRMS1_SX47 -MROA0_SA1 -MROA0_SA2 -MROA0_SI1307 -MROA0_SI1970 -MROA0_SI677 -MROA0_SX137 -MROA0_SX227 -MROA0_SX317 -MROA0_SX407 -MROA0_SX47 -MRPC0_SA1 -MRPC0_SA2 -MRPC0_SI1753 -MRPC0_SI493 -MRPC0_SI933 -MRPC0_SX133 -MRPC0_SX223 -MRPC0_SX313 -MRPC0_SX403 -MRPC0_SX43 -MRPP0_SA1 -MRPP0_SA2 -MRPP0_SI1184 -MRPP0_SI1814 -MRPP0_SI554 -MRPP0_SX104 -MRPP0_SX14 -MRPP0_SX194 -MRPP0_SX284 -MRPP0_SX374 -MRRK0_SA1 -MRRK0_SA2 -MRRK0_SI1288 -MRRK0_SI1716 -MRRK0_SI1918 -MRRK0_SX118 -MRRK0_SX208 -MRRK0_SX28 -MRRK0_SX298 -MRRK0_SX388 -MRTK0_SA1 -MRTK0_SA2 -MRTK0_SI1093 -MRTK0_SI1723 -MRTK0_SI1750 -MRTK0_SX103 -MRTK0_SX13 -MRTK0_SX193 -MRTK0_SX283 -MRTK0_SX373 -MRWS1_SA1 -MRWS1_SA2 -MRWS1_SI1130 -MRWS1_SI1496 -MRWS1_SI500 -MRWS1_SX140 -MRWS1_SX230 -MRWS1_SX320 -MRWS1_SX410 -MRWS1_SX50 -MSFH1_SA1 -MSFH1_SA2 -MSFH1_SI1270 -MSFH1_SI1900 -MSFH1_SI640 -MSFH1_SX10 -MSFH1_SX100 -MSFH1_SX190 -MSFH1_SX280 -MSFH1_SX370 -MSJS1_SA1 -MSJS1_SA2 -MSJS1_SI1899 -MSJS1_SI639 -MSJS1_SI869 -MSJS1_SX189 -MSJS1_SX279 -MSJS1_SX369 -MSJS1_SX9 -MSJS1_SX99 -MSLB0_SA1 -MSLB0_SA2 -MSLB0_SI1193 -MSLB0_SI1823 -MSLB0_SI563 -MSLB0_SX113 -MSLB0_SX203 -MSLB0_SX23 -MSLB0_SX293 -MSLB0_SX383 -MSTK0_SA1 -MSTK0_SA2 -MSTK0_SI1024 -MSTK0_SI2222 -MSTK0_SI2284 -MSTK0_SX124 -MSTK0_SX214 -MSTK0_SX304 -MSTK0_SX34 -MSTK0_SX394 -MTAA0_SA1 -MTAA0_SA2 -MTAA0_SI1285 -MTAA0_SI1915 -MTAA0_SI596 -MTAA0_SX115 -MTAA0_SX205 -MTAA0_SX25 -MTAA0_SX295 -MTAA0_SX385 -MTAS1_SA1 -MTAS1_SA2 -MTAS1_SI1473 -MTAS1_SI2098 -MTAS1_SI838 -MTAS1_SX118 -MTAS1_SX208 -MTAS1_SX28 -MTAS1_SX298 -MTAS1_SX388 -MTDT0_SA1 -MTDT0_SA2 -MTDT0_SI1994 -MTDT0_SI2254 -MTDT0_SI994 -MTDT0_SX184 -MTDT0_SX274 -MTDT0_SX364 -MTDT0_SX4 -MTDT0_SX94 -MTEB0_SA1 -MTEB0_SA2 -MTEB0_SI1133 -MTEB0_SI2064 -MTEB0_SI503 -MTEB0_SX143 -MTEB0_SX233 -MTEB0_SX323 -MTEB0_SX413 -MTEB0_SX53 -MTHC0_SA1 -MTHC0_SA2 -MTHC0_SI1015 -MTHC0_SI1645 -MTHC0_SI2275 -MTHC0_SX115 -MTHC0_SX205 -MTHC0_SX25 -MTHC0_SX295 -MTHC0_SX385 -MTLS0_SA1 -MTLS0_SA2 -MTLS0_SI1370 -MTLS0_SI2000 -MTLS0_SI740 -MTLS0_SX110 -MTLS0_SX20 -MTLS0_SX200 -MTLS0_SX290 -MTLS0_SX380 -MTMR0_SA1 -MTMR0_SA2 -MTMR0_SI1303 -MTMR0_SI1933 -MTMR0_SI673 -MTMR0_SX133 -MTMR0_SX223 -MTMR0_SX313 -MTMR0_SX403 -MTMR0_SX43 -MTWH0_SA1 -MTWH0_SA2 -MTWH0_SI1190 -MTWH0_SI1629 -MTWH0_SI1820 -MTWH0_SX110 -MTWH0_SX20 -MTWH0_SX200 -MTWH0_SX290 -MTWH0_SX380 -MWBT0_SA1 -MWBT0_SA2 -MWBT0_SI1553 -MWBT0_SI2183 -MWBT0_SI923 -MWBT0_SX113 -MWBT0_SX203 -MWBT0_SX23 -MWBT0_SX293 -MWBT0_SX383 -MWEW0_SA1 -MWEW0_SA2 -MWEW0_SI1361 -MWEW0_SI1991 -MWEW0_SI731 -MWEW0_SX101 -MWEW0_SX11 -MWEW0_SX191 -MWEW0_SX281 -MWEW0_SX371 -MWJG0_SA1 -MWJG0_SA2 -MWJG0_SI1124 -MWJG0_SI1754 -MWJG0_SI494 -MWJG0_SX134 -MWJG0_SX224 -MWJG0_SX314 -MWJG0_SX404 -MWJG0_SX44 -MWVW0_SA1 -MWVW0_SA2 -MWVW0_SI1476 -MWVW0_SI2106 -MWVW0_SI846 -MWVW0_SX126 -MWVW0_SX216 -MWVW0_SX306 -MWVW0_SX36 -MWVW0_SX396 diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/config/timit_unmatched/train.uid b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/config/timit_unmatched/train.uid deleted file mode 100644 index 35b02e7f82cc788f59860befad083ba8cfc899c0..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/config/timit_unmatched/train.uid +++ /dev/null @@ -1,3000 +0,0 @@ -FAEM0_SA1 -FAEM0_SA2 -FAEM0_SI2022 -FAEM0_SX132 -FAEM0_SX222 -FAEM0_SX312 -FAEM0_SX402 -FAJW0_SA2 -FAJW0_SI1893 -FAJW0_SX183 -FAJW0_SX273 -FAJW0_SX363 -FALK0_SA1 -FALK0_SA2 -FALK0_SI1086 -FALK0_SI456 -FALK0_SX276 -FALK0_SX366 -FALK0_SX96 -FALR0_SA1 -FALR0_SA2 -FALR0_SI1955 -FALR0_SI695 -FALR0_SX155 -FALR0_SX245 -FALR0_SX425 -FALR0_SX65 -FAPB0_SA1 -FAPB0_SA2 -FAPB0_SI1693 -FAPB0_SX163 -FAPB0_SX253 -FAPB0_SX343 -FAPB0_SX73 -FBAS0_SA2 -FBAS0_SI1387 -FBAS0_SX127 -FBAS0_SX307 -FBAS0_SX37 -FBAS0_SX397 -FBCG1_SA2 -FBCG1_SI1612 -FBCG1_SI2242 -FBCG1_SI982 -FBCG1_SX262 -FBCG1_SX82 -FBCH0_SA1 -FBCH0_SA2 -FBCH0_SI1586 -FBCH0_SI956 -FBCH0_SX146 -FBCH0_SX326 -FBCH0_SX56 -FBJL0_SA1 -FBJL0_SA2 -FBJL0_SI1552 -FBJL0_SI2182 -FBJL0_SX112 -FBJL0_SX202 -FBJL0_SX22 -FBJL0_SX292 -FBJL0_SX382 -FBLV0_SA2 -FBLV0_SI2318 -FBLV0_SX158 -FBLV0_SX248 -FBLV0_SX428 -FBMH0_SA2 -FBMH0_SI1766 -FBMH0_SX146 -FBMH0_SX236 -FBMH0_SX326 -FBMH0_SX416 -FBMH0_SX56 -FBMJ0_SA2 -FBMJ0_SX156 -FBMJ0_SX246 -FBMJ0_SX426 -FBMJ0_SX66 -FCAG0_SA2 -FCAG0_SI1503 -FCAG0_SI1641 -FCAG0_SI2133 -FCAG0_SX333 -FCAG0_SX423 -FCAG0_SX63 -FCAJ0_SA1 -FCAJ0_SA2 -FCAJ0_SI1804 -FCAJ0_SI849 -FCAJ0_SX129 -FCAJ0_SX219 -FCAJ0_SX39 -FCAJ0_SX399 -FCDR1_SA1 -FCDR1_SA2 -FCDR1_SX16 -FCDR1_SX376 -FCEG0_SA1 -FCEG0_SI1248 -FCEG0_SI1878 -FCEG0_SI618 -FCEG0_SX168 -FCEG0_SX258 -FCEG0_SX348 -FCEG0_SX438 -FCEG0_SX78 -FCJF0_SA2 -FCJF0_SI1027 -FCJF0_SI1657 -FCJF0_SI648 -FCJF0_SX217 -FCJF0_SX307 -FCJF0_SX37 -FCJF0_SX397 -FCJS0_SA1 -FCJS0_SA2 -FCJS0_SI977 -FCJS0_SX167 -FCJS0_SX347 -FCJS0_SX437 -FCJS0_SX77 -FCKE0_SA1 -FCKE0_SI1111 -FCKE0_SX211 -FCKE0_SX301 -FCKE0_SX31 -FCKE0_SX391 -FCLT0_SA1 -FCLT0_SA2 -FCLT0_SI1438 -FCLT0_SX178 -FCLT0_SX268 -FCLT0_SX358 -FCMG0_SA1 -FCMG0_SI1242 -FCMG0_SX162 -FCMG0_SX252 -FCMG0_SX342 -FCMM0_SI1083 -FCMM0_SI453 -FCMM0_SX273 -FCMM0_SX363 -FCMM0_SX93 -FCRZ0_SA1 -FCRZ0_SA2 -FCRZ0_SI1913 -FCRZ0_SI793 -FCRZ0_SX163 -FCRZ0_SX253 -FCRZ0_SX343 -FCRZ0_SX73 -FCYL0_SA2 -FCYL0_SI1297 -FCYL0_SI1927 -FCYL0_SX127 -FCYL0_SX217 -FCYL0_SX397 -FDAS1_SA1 -FDAS1_SA2 -FDAS1_SX111 -FDAS1_SX21 -FDAS1_SX291 -FDAW0_SA1 -FDAW0_SA2 -FDAW0_SX146 -FDAW0_SX236 -FDAW0_SX326 -FDAW0_SX416 -FDAW0_SX56 -FDFB0_SI1318 -FDFB0_SI1948 -FDFB0_SX148 -FDFB0_SX238 -FDFB0_SX328 -FDFB0_SX418 -FDJH0_SA1 -FDJH0_SA2 -FDJH0_SI1565 -FDJH0_SI2195 -FDJH0_SX125 -FDJH0_SX215 -FDJH0_SX35 -FDJH0_SX395 -FDKN0_SA1 -FDKN0_SA2 -FDKN0_SI1081 -FDKN0_SI1711 -FDKN0_SX271 -FDKN0_SX361 -FDKN0_SX91 -FDML0_SA1 -FDML0_SI1149 -FDML0_SI1779 -FDML0_SI2075 -FDML0_SX339 -FDML0_SX69 -FDMY0_SI1197 -FDMY0_SX117 -FDMY0_SX207 -FDMY0_SX297 -FDNC0_SA1 -FDNC0_SA2 -FDNC0_SI2287 -FDNC0_SX108 -FDNC0_SX18 -FDNC0_SX378 -FDTD0_SA2 -FDTD0_SI1561 -FDTD0_SI2191 -FDTD0_SI931 -FDTD0_SX121 -FDTD0_SX301 -FDTD0_SX391 -FDXW0_SA2 -FDXW0_SI1511 -FDXW0_SI2141 -FDXW0_SI881 -FDXW0_SX161 -FDXW0_SX431 -FEAC0_SA1 -FEAC0_SA2 -FEAC0_SI1245 -FEAC0_SI1875 -FEAC0_SX255 -FEAC0_SX345 -FEAC0_SX435 -FEAR0_SA1 -FEAR0_SA2 -FEAR0_SI1252 -FEAR0_SI1882 -FEAR0_SX172 -FEAR0_SX262 -FEAR0_SX442 -FEAR0_SX82 -FECD0_SA2 -FECD0_SI2048 -FECD0_SX158 -FECD0_SX248 -FECD0_SX338 -FECD0_SX428 -FEEH0_SA2 -FEEH0_SI1112 -FEEH0_SX212 -FEEH0_SX302 -FEEH0_SX32 -FEEH0_SX392 -FEME0_SA2 -FEME0_SI1505 -FEME0_SI2135 -FEME0_SX245 -FEME0_SX425 -FETB0_SA2 -FETB0_SI1778 -FETB0_SI518 -FETB0_SX248 -FETB0_SX338 -FETB0_SX428 -FETB0_SX68 -FEXM0_SA2 -FEXM0_SI1731 -FEXM0_SX111 -FEXM0_SX201 -FEXM0_SX291 -FEXM0_SX381 -FGCS0_SA1 -FGCS0_SA2 -FGCS0_SI1486 -FGCS0_SI2116 -FGCS0_SI856 -FGCS0_SX46 -FGDP0_SA2 -FGDP0_SI1618 -FGDP0_SI2248 -FGDP0_SX178 -FGDP0_SX268 -FGDP0_SX358 -FGDP0_SX448 -FGMB0_SA1 -FGMB0_SA2 -FGMB0_SI515 -FGMB0_SX155 -FGMB0_SX425 -FGMB0_SX65 -FGRW0_SA2 -FGRW0_SI1782 -FGRW0_SI1990 -FGRW0_SX252 -FGRW0_SX342 -FGRW0_SX72 -FHLM0_SA1 -FHLM0_SA2 -FHLM0_SI1560 -FHLM0_SI2190 -FHLM0_SI930 -FHLM0_SX210 -FHLM0_SX300 -FHXS0_SI2335 -FHXS0_SX265 -FHXS0_SX355 -FHXS0_SX85 -FJDM2_SI1582 -FJDM2_SI1964 -FJDM2_SI2212 -FJDM2_SX322 -FJDM2_SX412 -FJEN0_SA2 -FJEN0_SI1047 -FJEN0_SI1677 -FJEN0_SI2307 -FJEN0_SX147 -FJEN0_SX237 -FJEN0_SX57 -FJHK0_SA1 -FJHK0_SA2 -FJHK0_SI1022 -FJHK0_SI1652 -FJHK0_SX122 -FJHK0_SX212 -FJHK0_SX32 -FJHK0_SX392 -FJKL0_SA1 -FJKL0_SA2 -FJKL0_SI1562 -FJKL0_SI2192 -FJKL0_SX122 -FJKL0_SX302 -FJKL0_SX32 -FJLG0_SA1 -FJLG0_SA2 -FJLG0_SI1506 -FJLG0_SX179 -FJLG0_SX269 -FJLG0_SX359 -FJLG0_SX449 -FJLG0_SX89 -FJLR0_SA2 -FJLR0_SI1861 -FJLR0_SI601 -FJLR0_SX151 -FJLR0_SX241 -FJLR0_SX331 -FJLR0_SX421 -FJLR0_SX61 -FJRB0_SA1 -FJRB0_SA2 -FJRB0_SI1302 -FJRB0_SI1932 -FJRB0_SI672 -FJRB0_SX132 -FJRB0_SX222 -FJRB0_SX312 -FJRB0_SX42 -FJRP1_SA2 -FJRP1_SI802 -FJRP1_SX172 -FJRP1_SX442 -FJSK0_SA2 -FJSK0_SI1682 -FJSK0_SI2312 -FJSK0_SX152 -FJSK0_SX242 -FJSK0_SX332 -FJSK0_SX422 -FJSK0_SX62 -FJSP0_SA1 -FJSP0_SA2 -FJSP0_SI1763 -FJSP0_SI804 -FJSP0_SX174 -FJSP0_SX84 -FJWB1_SA2 -FJWB1_SI2055 -FJWB1_SI795 -FJWB1_SX165 -FJWB1_SX255 -FJWB1_SX75 -FJXM0_SA2 -FJXM0_SI1211 -FJXM0_SI1971 -FJXM0_SX131 -FJXM0_SX221 -FJXP0_SA2 -FJXP0_SI492 -FJXP0_SX222 -FJXP0_SX312 -FJXP0_SX402 -FJXP0_SX42 -FKAA0_SA2 -FKAA0_SI1208 -FKAA0_SI1838 -FKAA0_SI578 -FKAA0_SX218 -FKAA0_SX308 -FKAA0_SX38 -FKDE0_SA2 -FKDE0_SI2221 -FKDE0_SX331 -FKDW0_SA1 -FKDW0_SA2 -FKDW0_SI577 -FKDW0_SX127 -FKDW0_SX217 -FKDW0_SX307 -FKDW0_SX37 -FKFB0_SA1 -FKFB0_SI2238 -FKFB0_SI978 -FKFB0_SX168 -FKFB0_SX258 -FKKH0_SI660 -FKKH0_SX210 -FKKH0_SX30 -FKKH0_SX300 -FKLC0_SA1 -FKLC0_SA2 -FKLC0_SI1615 -FKLC0_SI2245 -FKLC0_SX265 -FKLC0_SX445 -FKLC0_SX85 -FKLC1_SA1 -FKLC1_SA2 -FKLC1_SI1678 -FKLC1_SX148 -FKLC1_SX58 -FKLH0_SA1 -FKLH0_SI1887 -FKLH0_SI627 -FKLH0_SX267 -FKLH0_SX357 -FKLH0_SX447 -FKLH0_SX87 -FKSR0_SI1117 -FKSR0_SX161 -FKSR0_SX37 -FKSR0_SX397 -FLAC0_SA1 -FLAC0_SA2 -FLAC0_SI2161 -FLAC0_SI901 -FLAC0_SX181 -FLAC0_SX271 -FLAC0_SX361 -FLAC0_SX91 -FLAG0_SA1 -FLAG0_SI2094 -FLAG0_SX294 -FLEH0_SA1 -FLEH0_SA2 -FLEH0_SX151 -FLEH0_SX241 -FLEH0_SX421 -FLEH0_SX61 -FLET0_SA2 -FLET0_SI1137 -FLET0_SI1767 -FLET0_SX147 -FLET0_SX237 -FLET0_SX277 -FLET0_SX417 -FLET0_SX57 -FLHD0_SA1 -FLHD0_SA2 -FLHD0_SI1344 -FLHD0_SI1974 -FLHD0_SX174 -FLHD0_SX264 -FLHD0_SX444 -FLHD0_SX84 -FLJA0_SA2 -FLJA0_SI1708 -FLJA0_SX268 -FLJA0_SX358 -FLJA0_SX448 -FLJA0_SX88 -FLJD0_SA1 -FLJD0_SA2 -FLJD0_SI2146 -FLJD0_SX166 -FLJD0_SX256 -FLJD0_SX346 -FLJD0_SX436 -FLJG0_SA1 -FLJG0_SI1611 -FLJG0_SI2241 -FLJG0_SX261 -FLJG0_SX441 -FLJG0_SX81 -FLKM0_SI1880 -FLKM0_SX116 -FLMA0_SA2 -FLMA0_SI1243 -FLMA0_SI1873 -FLMA0_SX163 -FLMA0_SX253 -FLMA0_SX343 -FLMC0_SA1 -FLMC0_SA2 -FLMC0_SI2002 -FLMC0_SI742 -FLMC0_SX112 -FLMC0_SX292 -FLMC0_SX336 -FLMC0_SX382 -FLMK0_SA2 -FLMK0_SI2295 -FLMK0_SX135 -FLMK0_SX225 -FLMK0_SX45 -FLOD0_SA1 -FLOD0_SA2 -FLOD0_SI1287 -FLOD0_SI657 -FLOD0_SX207 -FLOD0_SX387 -FLTM0_SA2 -FLTM0_SI1700 -FLTM0_SX260 -FLTM0_SX80 -FMAH1_SA1 -FMAH1_SI1509 -FMAH1_SI2139 -FMAH1_SX249 -FMAH1_SX339 -FMAH1_SX429 -FMAH1_SX69 -FMBG0_SA1 -FMBG0_SI1790 -FMBG0_SX260 -FMBG0_SX3 -FMBG0_SX350 -FMBG0_SX440 -FMBG0_SX80 -FMEM0_SA2 -FMEM0_SI1377 -FMEM0_SI2007 -FMEM0_SX117 -FMEM0_SX207 -FMEM0_SX297 -FMJB0_SA1 -FMJB0_SA2 -FMJB0_SI1807 -FMJB0_SX187 -FMJB0_SX277 -FMJB0_SX367 -FMJB0_SX7 -FMJF0_SA1 -FMJF0_SI1254 -FMJF0_SI1884 -FMJF0_SX264 -FMJF0_SX354 -FMJF0_SX444 -FMJU0_SA1 -FMJU0_SA2 -FMJU0_SI2019 -FMJU0_SI759 -FMJU0_SX129 -FMJU0_SX219 -FMJU0_SX39 -FMKC0_SA1 -FMKC0_SA2 -FMKC0_SI1072 -FMKC0_SX172 -FMKC0_SX262 -FMKC0_SX352 -FMKF0_SA1 -FMKF0_SA2 -FMKF0_SI1536 -FMKF0_SI906 -FMKF0_SX276 -FMKF0_SX366 -FMKF0_SX6 -FMKF0_SX96 -FMMH0_SA1 -FMMH0_SA2 -FMMH0_SI1537 -FMMH0_SI2167 -FMMH0_SI907 -FMMH0_SX187 -FMMH0_SX367 -FMMH0_SX420 -FMMH0_SX7 -FMMH0_SX97 -FMPG0_SI1602 -FMPG0_SI2232 -FMPG0_SX252 -FMPG0_SX72 -FNKL0_SA1 -FNKL0_SA2 -FNKL0_SI2152 -FNKL0_SX172 -FNKL0_SX196 -FNKL0_SX262 -FNKL0_SX442 -FNKL0_SX82 -FNTB0_SA1 -FNTB0_SA2 -FNTB0_SX123 -FNTB0_SX213 -FNTB0_SX33 -FNTB0_SX393 -FPAB1_SA2 -FPAB1_SX121 -FPAB1_SX301 -FPAB1_SX31 -FPAB1_SX391 -FPAC0_SA1 -FPAC0_SI2011 -FPAC0_SX121 -FPAC0_SX211 -FPAC0_SX301 -FPAC0_SX31 -FPAC0_SX391 -FPAD0_SA1 -FPAD0_SI1346 -FPAD0_SI1976 -FPAD0_SX266 -FPAD0_SX446 -FPAF0_SI1684 -FPAF0_SI2314 -FPAF0_SX244 -FPAF0_SX334 -FPAF0_SX424 -FPAF0_SX64 -FPAZ0_SI1593 -FPAZ0_SX153 -FPAZ0_SX27 -FPAZ0_SX423 -FPAZ0_SX63 -FPJF0_SA2 -FPJF0_SI1046 -FPJF0_SI1676 -FPJF0_SX236 -FPJF0_SX326 -FPLS0_SA1 -FPLS0_SA2 -FPLS0_SI2220 -FPLS0_SX150 -FPLS0_SX240 -FPLS0_SX3 -FPLS0_SX60 -FPMY0_SA2 -FPMY0_SI1783 -FPMY0_SX163 -FPMY0_SX196 -FPMY0_SX253 -FPMY0_SX73 -FREH0_SI1315 -FREH0_SI685 -FREH0_SX145 -FREH0_SX235 -FREH0_SX325 -FREH0_SX55 -FRJB0_SA1 -FRJB0_SA2 -FRJB0_SI1427 -FRJB0_SI1470 -FRJB0_SI1794 -FRJB0_SX167 -FRJB0_SX257 -FRJB0_SX437 -FRJB0_SX77 -FRLL0_SA1 -FRLL0_SA2 -FRLL0_SI1514 -FRLL0_SI884 -FRLL0_SX164 -FRLL0_SX254 -FRLL0_SX344 -FRLL0_SX74 -FSAG0_SA2 -FSAG0_SI1953 -FSAG0_SI693 -FSAG0_SX63 -FSAH0_SI1244 -FSAH0_SI1874 -FSAH0_SX344 -FSAH0_SX74 -FSAK0_SA1 -FSAK0_SA2 -FSAK0_SI1930 -FSAK0_SI670 -FSAK0_SX130 -FSAK0_SX220 -FSAK0_SX310 -FSAK0_SX40 -FSAK0_SX400 -FSBK0_SA1 -FSBK0_SI1699 -FSBK0_SI2329 -FSBK0_SX259 -FSBK0_SX439 -FSBK0_SX79 -FSCN0_SI1886 -FSCN0_SX356 -FSDC0_SA1 -FSDC0_SI1942 -FSDC0_SI2234 -FSDC0_SX232 -FSDC0_SX412 -FSDJ0_SA1 -FSDJ0_SA2 -FSDJ0_SI1745 -FSDJ0_SX125 -FSDJ0_SX35 -FSGF0_SA1 -FSGF0_SA2 -FSGF0_SI1557 -FSGF0_SX207 -FSGF0_SX27 -FSGF0_SX297 -FSGF0_SX387 -FSJG0_SI1570 -FSJG0_SI2200 -FSJG0_SX310 -FSJK1_SA1 -FSJK1_SI1025 -FSJK1_SI2285 -FSJK1_SI696 -FSJK1_SX215 -FSJK1_SX305 -FSJK1_SX395 -FSJS0_SA2 -FSJS0_SI1171 -FSJS0_SI1801 -FSJS0_SI541 -FSJS0_SX271 -FSJS0_SX361 -FSJS0_SX91 -FSJW0_SA1 -FSJW0_SA2 -FSJW0_SI703 -FSJW0_SX163 -FSJW0_SX253 -FSJW0_SX343 -FSJW0_SX73 -FSKC0_SA1 -FSKC0_SA2 -FSKC0_SI2046 -FSKC0_SX156 -FSKC0_SX336 -FSKC0_SX426 -FSKC0_SX66 -FSKL0_SA1 -FSKL0_SA2 -FSKL0_SI2159 -FSKL0_SI899 -FSKL0_SX179 -FSKL0_SX269 -FSKL0_SX359 -FSKL0_SX89 -FSKP0_SA1 -FSKP0_SI1728 -FSKP0_SI468 -FSKP0_SX108 -FSKP0_SX18 -FSKP0_SX198 -FSKP0_SX288 -FSKP0_SX378 -FSLS0_SA1 -FSLS0_SA2 -FSLS0_SI1056 -FSLS0_SI1686 -FSLS0_SI2316 -FSLS0_SX202 -FSLS0_SX246 -FSLS0_SX66 -FSMA0_SA1 -FSMA0_SI1621 -FSMA0_SI2251 -FSMA0_SX271 -FSMA0_SX361 -FSMA0_SX91 -FSMM0_SA1 -FSMM0_SA2 -FSMM0_SI1314 -FSMM0_SI1944 -FSMM0_SI684 -FSMM0_SX414 -FSMM0_SX54 -FSMS1_SA1 -FSMS1_SA2 -FSMS1_SI1504 -FSMS1_SI2134 -FSMS1_SI874 -FSMS1_SX154 -FSMS1_SX334 -FSMS1_SX64 -FSPM0_SA1 -FSPM0_SI1871 -FSPM0_SI611 -FSPM0_SX341 -FSPM0_SX431 -FSRH0_SA1 -FSRH0_SA2 -FSRH0_SI1719 -FSRH0_SX131 -FSRH0_SX41 -FSSB0_SA1 -FSSB0_SA2 -FSSB0_SI1082 -FSSB0_SI2342 -FSSB0_SX182 -FSSB0_SX272 -FSSB0_SX452 -FSSB0_SX92 -FTAJ0_SA1 -FTAJ0_SA2 -FTAJ0_SI1329 -FTAJ0_SI474 -FTAJ0_SX339 -FTAJ0_SX69 -FTBR0_SA1 -FTBR0_SA2 -FTBR0_SI2181 -FTBR0_SX111 -FTBR0_SX201 -FTBR0_SX291 -FTBR0_SX381 -FTBW0_SA2 -FTBW0_SI1345 -FTBW0_SI1975 -FTBW0_SX265 -FTBW0_SX355 -FTBW0_SX445 -FTBW0_SX85 -FTLG0_SA1 -FTLG0_SA2 -FTLG0_SI840 -FTLG0_SX123 -FTLG0_SX213 -FTLG0_SX303 -FTLG0_SX33 -FTLG0_SX393 -FTMG0_SA1 -FTMG0_SA2 -FTMG0_SX182 -FTMG0_SX272 -FTMG0_SX362 -FTMG0_SX92 -FVFB0_SA1 -FVFB0_SI1032 -FVFB0_SI2292 -FVFB0_SX222 -FVFB0_SX312 -FVFB0_SX402 -FVKB0_SA2 -FVKB0_SI1159 -FVKB0_SI1789 -FVKB0_SI529 -FVKB0_SX169 -FVKB0_SX259 -FVKB0_SX439 -FVKB0_SX79 -FVMH0_SA1 -FVMH0_SI2096 -FVMH0_SX206 -FVMH0_SX296 -FVMH0_SX386 -MABC0_SA1 -MABC0_SA2 -MABC0_SX151 -MABC0_SX241 -MABC0_SX331 -MABC0_SX421 -MABC0_SX61 -MADC0_SA1 -MADC0_SA2 -MADC0_SI1997 -MADC0_SX17 -MADC0_SX197 -MADC0_SX287 -MADD0_SA1 -MADD0_SI1798 -MADD0_SI538 -MADD0_SX358 -MADD0_SX448 -MAEB0_SA1 -MAEB0_SA2 -MAEB0_SI2250 -MAEB0_SI990 -MAEB0_SX180 -MAEB0_SX270 -MAEB0_SX360 -MAEB0_SX90 -MAEO0_SA2 -MAEO0_SI1655 -MAEO0_SI1956 -MAEO0_SX156 -MAEO0_SX246 -MAEO0_SX336 -MAEO0_SX426 -MAEO0_SX66 -MAFM0_SA1 -MAFM0_SA2 -MAFM0_SI1569 -MAFM0_SI2199 -MAFM0_SX219 -MAFM0_SX39 -MAFM0_SX399 -MAJP0_SA1 -MAJP0_SI1074 -MAJP0_SI2334 -MAJP0_SX264 -MAJP0_SX354 -MAJP0_SX444 -MAJP0_SX84 -MAKB0_SA1 -MAKB0_SX206 -MAKB0_SX296 -MAKR0_SA1 -MAKR0_SA2 -MAKR0_SI1352 -MAKR0_SI1982 -MAKR0_SI722 -MAKR0_SX182 -MAKR0_SX272 -MAKR0_SX452 -MAPV0_SA1 -MAPV0_SA2 -MAPV0_SI1923 -MAPV0_SX123 -MAPV0_SX303 -MAPV0_SX33 -MAPV0_SX393 -MARC0_SA1 -MARC0_SI1188 -MARC0_SI1818 -MARC0_SI558 -MARC0_SX288 -MARC0_SX378 -MARW0_SA1 -MARW0_SA2 -MARW0_SI1276 -MARW0_SI646 -MARW0_SX106 -MARW0_SX16 -MARW0_SX376 -MBAR0_SA2 -MBAR0_SI1319 -MBAR0_SI1949 -MBAR0_SI689 -MBAR0_SX149 -MBAR0_SX239 -MBAR0_SX329 -MBBR0_SA1 -MBBR0_SA2 -MBBR0_SI1685 -MBBR0_SX155 -MBBR0_SX245 -MBBR0_SX425 -MBCG0_SA2 -MBCG0_SI2217 -MBCG0_SX147 -MBCG0_SX237 -MBCG0_SX417 -MBCG0_SX57 -MBEF0_SA1 -MBEF0_SA2 -MBEF0_SX111 -MBEF0_SX201 -MBEF0_SX291 -MBGT0_SA1 -MBGT0_SI1341 -MBGT0_SI711 -MBGT0_SX81 -MBJV0_SA2 -MBJV0_SI1247 -MBJV0_SI1877 -MBJV0_SX167 -MBJV0_SX257 -MBJV0_SX437 -MBJV0_SX77 -MBMA0_SA1 -MBMA0_SA2 -MBMA0_SI1852 -MBMA0_SX142 -MBMA0_SX322 -MBMA0_SX412 -MBMA1_SA1 -MBMA1_SA2 -MBMA1_SI2207 -MBMA1_SX144 -MBMA1_SX234 -MBMA1_SX414 -MBML0_SA1 -MBML0_SI1799 -MBML0_SI539 -MBML0_SX179 -MBML0_SX269 -MBML0_SX359 -MBML0_SX449 -MBOM0_SA1 -MBOM0_SI1014 -MBOM0_SI1644 -MBOM0_SX114 -MBOM0_SX204 -MBOM0_SX311 -MBOM0_SX384 -MBSB0_SA2 -MBSB0_SI1353 -MBSB0_SI1983 -MBSB0_SI723 -MBSB0_SX183 -MBSB0_SX273 -MBSB0_SX363 -MBSB0_SX93 -MBTH0_SA1 -MBTH0_SI505 -MBTH0_SI757 -MBTH0_SX212 -MBTH0_SX302 -MBTH0_SX392 -MBWP0_SA1 -MBWP0_SA2 -MBWP0_SI1531 -MBWP0_SI1969 -MBWP0_SI709 -MBWP0_SX169 -MBWP0_SX259 -MBWP0_SX439 -MBWP0_SX79 -MCAE0_SA1 -MCAE0_SA2 -MCAE0_SX187 -MCAE0_SX367 -MCAE0_SX7 -MCAE0_SX97 -MCAL0_SA1 -MCAL0_SI508 -MCAL0_SX148 -MCAL0_SX238 -MCAL0_SX328 -MCAL0_SX418 -MCAL0_SX58 -MCDC0_SA2 -MCDC0_SI1292 -MCDC0_SI1922 -MCDC0_SI662 -MCDC0_SX122 -MCDC0_SX302 -MCDC0_SX32 -MCDC0_SX392 -MCDD0_SA1 -MCDD0_SI1513 -MCDD0_SI2143 -MCDD0_SX163 -MCDD0_SX343 -MCDD0_SX73 -MCDR0_SA1 -MCDR0_SA2 -MCDR0_SX164 -MCDR0_SX254 -MCDR0_SX344 -MCDR0_SX434 -MCDR0_SX74 -MCEF0_SA1 -MCEF0_SA2 -MCEF0_SI1135 -MCEF0_SI1765 -MCEF0_SX145 -MCEF0_SX325 -MCEF0_SX55 -MCEW0_SI1442 -MCEW0_SX182 -MCEW0_SX272 -MCEW0_SX92 -MCHL0_SA1 -MCHL0_SA2 -MCHL0_SI1977 -MCHL0_SX177 -MCHL0_SX267 -MCHL0_SX357 -MCHL0_SX447 -MCLK0_SA1 -MCLK0_SA2 -MCLK0_SI1660 -MCLK0_SX130 -MCLK0_SX220 -MCLK0_SX40 -MCLK0_SX400 -MCLM0_SA2 -MCLM0_SI1456 -MCLM0_SX106 -MCLM0_SX16 -MCLM0_SX196 -MCLM0_SX286 -MCLM0_SX376 -MCPM0_SA2 -MCPM0_SI1194 -MCPM0_SI564 -MCPM0_SX204 -MCPM0_SX24 -MCRE0_SA1 -MCRE0_SA2 -MCRE0_SI1121 -MCRE0_SI1725 -MCRE0_SI1751 -MCRE0_SX131 -MCRE0_SX221 -MCRE0_SX24 -MCRE0_SX401 -MCRE0_SX41 -MCSS0_SA1 -MCSS0_SA2 -MCSS0_SX120 -MCSS0_SX210 -MCSS0_SX30 -MCSS0_SX300 -MCSS0_SX390 -MCTH0_SA2 -MCTH0_SI1209 -MCTH0_SI1839 -MCTH0_SI579 -MCTH0_SX129 -MCTH0_SX219 -MCTH0_SX309 -MCTH0_SX399 -MCTM0_SA1 -MCTM0_SA2 -MCTM0_SI720 -MCTM0_SX180 -MCTM0_SX270 -MCTM0_SX360 -MCTM0_SX450 -MCTM0_SX90 -MCXM0_SA1 -MCXM0_SA2 -MCXM0_SI1351 -MCXM0_SI1981 -MCXM0_SI721 -MCXM0_SX181 -MCXM0_SX271 -MCXM0_SX361 -MCXM0_SX451 -MDAC0_SA2 -MDAC0_SI1261 -MDAC0_SI1837 -MDAC0_SX271 -MDAC0_SX451 -MDAC0_SX91 -MDAS0_SA1 -MDAS0_SA2 -MDAS0_SI1266 -MDAS0_SX186 -MDAS0_SX21 -MDAS0_SX276 -MDAS0_SX96 -MDBB1_SA1 -MDBB1_SA2 -MDBB1_SI1006 -MDBB1_SI1636 -MDBB1_SI2056 -MDBB1_SX196 -MDBB1_SX286 -MDBP0_SA1 -MDBP0_SA2 -MDBP0_SI1158 -MDBP0_SI1788 -MDBP0_SX258 -MDBP0_SX348 -MDBP0_SX78 -MDCD0_SA1 -MDCD0_SA2 -MDCD0_SI2045 -MDCD0_SX155 -MDCD0_SX65 -MDCM0_SA1 -MDCM0_SA2 -MDCM0_SI2110 -MDCM0_SI850 -MDCM0_SX130 -MDCM0_SX220 -MDCM0_SX310 -MDDC0_SA1 -MDDC0_SA2 -MDDC0_SX249 -MDDC0_SX339 -MDDC0_SX429 -MDED0_SI1170 -MDED0_SI1800 -MDED0_SX180 -MDED0_SX270 -MDED0_SX360 -MDED0_SX450 -MDED0_SX90 -MDEF0_SA1 -MDEF0_SA2 -MDEF0_SI1563 -MDEF0_SI2193 -MDEF0_SX213 -MDEF0_SX33 -MDEF0_SX393 -MDEM0_SA2 -MDEM0_SI1868 -MDEM0_SX158 -MDEM0_SX248 -MDEM0_SX338 -MDEM0_SX68 -MDHL0_SA1 -MDHL0_SA2 -MDHL0_SI2069 -MDHL0_SI809 -MDHL0_SX179 -MDHL0_SX359 -MDHL0_SX89 -MDHS0_SX180 -MDHS0_SX270 -MDHS0_SX360 -MDHS0_SX450 -MDHS0_SX90 -MDJM0_SA1 -MDJM0_SA2 -MDJM0_SI2085 -MDJM0_SI825 -MDJM0_SX195 -MDJM0_SX285 -MDJM0_SX375 -MDKS0_SA1 -MDKS0_SA2 -MDKS0_SI1066 -MDKS0_SI1696 -MDKS0_SI2326 -MDKS0_SX256 -MDKS0_SX76 -MDLB0_SA1 -MDLB0_SI1936 -MDLB0_SI676 -MDLB0_SX226 -MDLB0_SX316 -MDLB0_SX46 -MDLC0_SA1 -MDLC0_SA2 -MDLC0_SI765 -MDLC0_SX135 -MDLC0_SX225 -MDLC0_SX315 -MDLC0_SX45 -MDLC1_SA1 -MDLC1_SX175 -MDLC1_SX265 -MDLC1_SX355 -MDLC1_SX85 -MDLC2_SA1 -MDLC2_SA2 -MDLC2_SI1614 -MDLC2_SI984 -MDLC2_SX174 -MDLC2_SX264 -MDLC2_SX444 -MDLC2_SX84 -MDLH0_SA1 -MDLH0_SI1960 -MDLH0_SI574 -MDLH0_SI700 -MDLH0_SX250 -MDLH0_SX340 -MDLH0_SX70 -MDLM0_SA1 -MDLM0_SA2 -MDLM0_SX244 -MDLM0_SX334 -MDLM0_SX64 -MDLR0_SI1233 -MDLR0_SX243 -MDLR0_SX423 -MDLR0_SX63 -MDLR1_SI1299 -MDLR1_SI1929 -MDLR1_SX129 -MDLR1_SX219 -MDLR1_SX309 -MDLR1_SX39 -MDLR1_SX399 -MDMA0_SA1 -MDMA0_SA2 -MDMA0_SI1238 -MDMA0_SI2060 -MDMT0_SI2341 -MDMT0_SI572 -MDMT0_SX212 -MDMT0_SX302 -MDMT0_SX392 -MDNS0_SA1 -MDNS0_SX111 -MDNS0_SX291 -MDNS0_SX381 -MDPB0_SA1 -MDPB0_SA2 -MDPB0_SI2126 -MDPB0_SX146 -MDPB0_SX236 -MDPB0_SX326 -MDPB0_SX56 -MDPK0_SA1 -MDPK0_SA2 -MDPK0_SI1683 -MDPK0_SI552 -MDPK0_SX153 -MDPK0_SX243 -MDPK0_SX63 -MDPS0_SA1 -MDPS0_SA2 -MDPS0_SI1651 -MDPS0_SI1979 -MDPS0_SX179 -MDPS0_SX269 -MDPS0_SX449 -MDPS0_SX89 -MDRD0_SA2 -MDRD0_SI1382 -MDRD0_SI2012 -MDRD0_SX122 -MDRD0_SX212 -MDRD0_SX302 -MDRD0_SX392 -MDSJ0_SA1 -MDSJ0_SA2 -MDSJ0_SI832 -MDSJ0_SX112 -MDSJ0_SX22 -MDSJ0_SX292 -MDSJ0_SX382 -MDSS0_SA1 -MDSS0_SI1881 -MDSS0_SI2087 -MDSS0_SI621 -MDSS0_SX171 -MDSS0_SX261 -MDSS0_SX351 -MDSS0_SX81 -MDSS1_SA2 -MDSS1_SI1713 -MDSS1_SX247 -MDSS1_SX337 -MDSS1_SX427 -MDTB0_SA1 -MDTB0_SA2 -MDTB0_SI570 -MDTB0_SX210 -MDTB0_SX300 -MDTB0_SX321 -MDTB0_SX390 -MDWD0_SA1 -MDWD0_SI1890 -MDWD0_SI557 -MDWD0_SX180 -MDWD0_SX360 -MDWD0_SX450 -MDWH0_SA2 -MDWH0_SI1925 -MDWH0_SX125 -MDWH0_SX35 -MDWH0_SX395 -MDWM0_SI1546 -MDWM0_SI2176 -MDWM0_SX106 -MDWM0_SX376 -MDWM0_SX433 -MEAL0_SA1 -MEAL0_SI1547 -MEAL0_SI917 -MEAL0_SX197 -MEAL0_SX287 -MEAL0_SX377 -MEDR0_SI744 -MEDR0_SX114 -MEDR0_SX204 -MEDR0_SX24 -MEDR0_SX294 -MEDR0_SX384 -MEFG0_SA2 -MEFG0_SI465 -MEFG0_SX105 -MEFG0_SX15 -MEFG0_SX195 -MEFG0_SX285 -MEFG0_SX375 -MEGJ0_SI1967 -MEGJ0_SX437 -MEGJ0_SX77 -MEJL0_SA2 -MEJL0_SI1592 -MEJL0_SI1654 -MEJL0_SI962 -MEJL0_SX332 -MEJL0_SX422 -MEJL0_SX62 -MEJS0_SA1 -MEJS0_SA2 -MEJS0_SI1870 -MEJS0_SX250 -MEJS0_SX430 -MEJS0_SX70 -MESG0_SA1 -MESG0_SA2 -MESG0_SI1332 -MESG0_SI1962 -MESG0_SX162 -MESG0_SX252 -MESG0_SX342 -MESG0_SX72 -MESJ0_SA1 -MESJ0_SA2 -MESJ0_SI2257 -MESJ0_SI997 -MESJ0_SX277 -MESJ0_SX367 -MESJ0_SX7 -MEWM0_SA1 -MEWM0_SA2 -MEWM0_SI1348 -MEWM0_SI1978 -MEWM0_SX268 -MEWM0_SX358 -MEWM0_SX448 -MFER0_SA1 -MFER0_SA2 -MFER0_SI1492 -MFER0_SI2122 -MFER0_SX232 -MFER0_SX322 -MFER0_SX412 -MFER0_SX52 -MFMC0_SA1 -MFMC0_SA2 -MFMC0_SI1132 -MFMC0_SI1762 -MFMC0_SI502 -MFMC0_SX142 -MFMC0_SX232 -MFMC0_SX322 -MFMC0_SX412 -MFMC0_SX52 -MFRM0_SA1 -MFRM0_SA2 -MFRM0_SI1155 -MFRM0_SI1717 -MFRM0_SI1785 -MFRM0_SX165 -MFRM0_SX255 -MFRM0_SX75 -MFWK0_SA1 -MFWK0_SA2 -MFWK0_SI1249 -MFWK0_SI619 -MFWK0_SX259 -MFWK0_SX439 -MFWK0_SX79 -MFXS0_SA1 -MFXS0_SA2 -MFXS0_SI1674 -MFXS0_SI2225 -MFXS0_SI2304 -MFXS0_SX144 -MFXS0_SX234 -MFXS0_SX414 -MFXV0_SA1 -MFXV0_SI1635 -MFXV0_SX15 -MFXV0_SX195 -MFXV0_SX285 -MFXV0_SX375 -MGAF0_SA2 -MGAF0_SI1912 -MGAF0_SI652 -MGAF0_SX112 -MGAF0_SX202 -MGAF0_SX292 -MGAG0_SA1 -MGAG0_SI1321 -MGAG0_SI645 -MGAG0_SX151 -MGAG0_SX241 -MGAG0_SX331 -MGAG0_SX421 -MGAG0_SX61 -MGAK0_SA1 -MGAK0_SA2 -MGAK0_SI1666 -MGAK0_SI2296 -MGAK0_SX316 -MGAK0_SX406 -MGAR0_SA1 -MGAR0_SA2 -MGAR0_SI1212 -MGAR0_SI1694 -MGAR0_SI1842 -MGAR0_SX222 -MGAR0_SX402 -MGAR0_SX42 -MGAW0_SA1 -MGAW0_SA2 -MGAW0_SI1802 -MGAW0_SX265 -MGAW0_SX355 -MGAW0_SX445 -MGAW0_SX85 -MGES0_SA2 -MGES0_SI1481 -MGES0_SX131 -MGES0_SX221 -MGES0_SX401 -MGES0_SX41 -MGJC0_SA1 -MGJC0_SI1256 -MGJC0_SI1335 -MGJC0_SI1965 -MGJC0_SX165 -MGJC0_SX255 -MGJC0_SX345 -MGRL0_SA1 -MGRL0_SA2 -MGRL0_SI1497 -MGRL0_SX237 -MGRL0_SX417 -MGRL0_SX57 -MGRP0_SA1 -MGRP0_SI1947 -MGRP0_SI687 -MGRP0_SX147 -MGRP0_SX237 -MGRP0_SX417 -MGRP0_SX57 -MGSH0_SA1 -MGSH0_SX186 -MGSH0_SX96 -MGSL0_SA2 -MGSL0_SI1164 -MGSL0_SX174 -MGSL0_SX354 -MGSL0_SX444 -MGSL0_SX84 -MGXP0_SA1 -MGXP0_SA2 -MGXP0_SI457 -MGXP0_SX277 -MGXP0_SX367 -MGXP0_SX97 -MHBS0_SA1 -MHBS0_SA2 -MHBS0_SI1575 -MHBS0_SI2205 -MHBS0_SX135 -MHBS0_SX225 -MHBS0_SX405 -MHIT0_SA2 -MHIT0_SI1613 -MHIT0_SI2243 -MHIT0_SX173 -MHIT0_SX263 -MHIT0_SX353 -MHIT0_SX443 -MHIT0_SX83 -MHJB0_SA2 -MHJB0_SI1647 -MHJB0_SI2277 -MHJB0_SX117 -MHJB0_SX207 -MHJB0_SX27 -MHJB0_SX297 -MHJB0_SX387 -MHMG0_SA1 -MHMG0_SA2 -MHMG0_SI1365 -MHMG0_SI1995 -MHMG0_SX105 -MHMG0_SX15 -MHMG0_SX285 -MHMG0_SX375 -MHMR0_SA2 -MHMR0_SI1119 -MHMR0_SX129 -MHMR0_SX219 -MHMR0_SX309 -MHMR0_SX39 -MHMR0_SX399 -MHRM0_SA2 -MHRM0_SI1475 -MHRM0_SI2218 -MHRM0_SX238 -MHRM0_SX328 -MHRM0_SX418 -MHXL0_SA1 -MHXL0_SA2 -MHXL0_SI512 -MHXL0_SI612 -MHXL0_SX152 -MHXL0_SX332 -MHXL0_SX422 -MHXL0_SX62 -MILB0_SA1 -MILB0_SI2163 -MILB0_SI807 -MILB0_SX183 -MILB0_SX273 -MILB0_SX3 -MILB0_SX363 -MILB0_SX93 -MJAC0_SA1 -MJAC0_SA2 -MJAC0_SI1331 -MJAC0_SI2148 -MJAC0_SX341 -MJAC0_SX431 -MJAE0_SA1 -MJAE0_SA2 -MJAE0_SI1524 -MJAE0_SI1999 -MJAE0_SI2154 -MJAE0_SX264 -MJAE0_SX354 -MJAE0_SX444 -MJAI0_SI1604 -MJAI0_SX164 -MJAI0_SX254 -MJAI0_SX344 -MJAI0_SX434 -MJAI0_SX74 -MJBG0_SA1 -MJBG0_SA2 -MJBG0_SI1232 -MJBG0_SI1724 -MJBG0_SI1862 -MJBG0_SX152 -MJBG0_SX242 -MJBG0_SX332 -MJBG0_SX422 -MJDA0_SA1 -MJDA0_SA2 -MJDA0_SI1661 -MJDA0_SI2291 -MJDA0_SX131 -MJDA0_SX221 -MJDA0_SX401 -MJDA0_SX41 -MJDC0_SA1 -MJDC0_SA2 -MJDC0_SI1161 -MJDC0_SI2165 -MJDC0_SX171 -MJDC0_SX261 -MJDC0_SX351 -MJDC0_SX441 -MJDC0_SX81 -MJDE0_SA2 -MJDE0_SX130 -MJDE0_SX310 -MJDE0_SX40 -MJDE0_SX400 -MJDG0_SA1 -MJDG0_SI1672 -MJDG0_SX142 -MJDG0_SX232 -MJDG0_SX322 -MJDG0_SX412 -MJDG0_SX52 -MJDM0_SA2 -MJDM0_SI1937 -MJDM0_SX260 -MJDM0_SX440 -MJDM0_SX80 -MJEB0_SA1 -MJEB0_SA2 -MJEB0_SI1286 -MJEB0_SI1916 -MJEB0_SX206 -MJEB0_SX26 -MJEB0_SX386 -MJEB1_SA1 -MJEB1_SI2097 -MJEB1_SX117 -MJEB1_SX27 -MJEB1_SX297 -MJEE0_SA2 -MJEE0_SI1237 -MJEE0_SI1867 -MJEE0_SI607 -MJEE0_SX157 -MJEE0_SX427 -MJEE0_SX67 -MJFH0_SA1 -MJFH0_SI1737 -MJFH0_SI477 -MJFH0_SX117 -MJFH0_SX207 -MJFH0_SX27 -MJFH0_SX297 -MJFH0_SX387 -MJFR0_SA2 -MJFR0_SI1605 -MJFR0_SI2235 -MJFR0_SI975 -MJFR0_SX165 -MJFR0_SX255 -MJFR0_SX345 -MJHI0_SA2 -MJHI0_SI555 -MJHI0_SI698 -MJHI0_SX248 -MJHI0_SX338 -MJHI0_SX428 -MJHI0_SX68 -MJJB0_SA2 -MJJB0_SI1139 -MJJB0_SI1277 -MJJB0_SI1769 -MJJB0_SX149 -MJJB0_SX329 -MJJB0_SX419 -MJJB0_SX59 -MJJJ0_SA1 -MJJJ0_SA2 -MJJJ0_SI1793 -MJJJ0_SI533 -MJJJ0_SX173 -MJJJ0_SX263 -MJJJ0_SX353 -MJJJ0_SX83 -MJJM0_SA1 -MJJM0_SI1457 -MJJM0_SX17 -MJJM0_SX197 -MJJM0_SX287 -MJJM0_SX377 -MJKR0_SA2 -MJKR0_SI1201 -MJKR0_SI1831 -MJKR0_SX121 -MJKR0_SX211 -MJKR0_SX301 -MJKR0_SX31 -MJKR0_SX391 -MJLB0_SA1 -MJLB0_SA2 -MJLB0_SI2246 -MJLB0_SI986 -MJLB0_SX266 -MJLB0_SX356 -MJLB0_SX446 -MJLB0_SX86 -MJLG1_SA1 -MJLG1_SA2 -MJLG1_SI1012 -MJLG1_SI1642 -MJLG1_SI2272 -MJLG1_SX112 -MJLG1_SX202 -MJLG1_SX22 -MJLG1_SX382 -MJLS0_SA1 -MJLS0_SA2 -MJLS0_SI1096 -MJLS0_SI466 -MJLS0_SX16 -MJLS0_SX196 -MJLS0_SX286 -MJLS0_SX376 -MJMA0_SI1495 -MJMA0_SI865 -MJMA0_SX145 -MJMA0_SX235 -MJMA0_SX325 -MJMA0_SX415 -MJMA0_SX55 -MJMD0_SA1 -MJMD0_SI1028 -MJMD0_SI1658 -MJMD0_SX128 -MJMD0_SX218 -MJMD0_SX398 -MJMM0_SA1 -MJMM0_SA2 -MJMM0_SI1885 -MJMM0_SI625 -MJMM0_SX265 -MJMM0_SX355 -MJMM0_SX445 -MJPG0_SA1 -MJPG0_SA2 -MJPG0_SI561 -MJPG0_SX291 -MJPG0_SX381 -MJPM0_SA1 -MJPM0_SI1998 -MJPM0_SI738 -MJPM0_SX108 -MJPM0_SX18 -MJPM0_SX198 -MJPM0_SX288 -MJPM1_SA1 -MJPM1_SA2 -MJPM1_SI1897 -MJPM1_SI761 -MJPM1_SX131 -MJPM1_SX221 -MJPM1_SX41 -MJRA0_SI606 -MJRA0_SX156 -MJRA0_SX246 -MJRA0_SX66 -MJRG0_SA1 -MJRG0_SA2 -MJRG0_SX106 -MJRG0_SX16 -MJRG0_SX286 -MJRH0_SA1 -MJRH0_SA2 -MJRH0_SI1125 -MJRH0_SI1755 -MJRH0_SX135 -MJRH0_SX315 -MJRH0_SX405 -MJRH0_SX45 -MJRH1_SA2 -MJRH1_SI1774 -MJRH1_SX334 -MJRH1_SX64 -MJRK0_SI2103 -MJRK0_SX340 -MJRK0_SX70 -MJRP0_SI1835 -MJRP0_SI585 -MJRP0_SX135 -MJRP0_SX315 -MJRP0_SX405 -MJRP0_SX45 -MJSR0_SA2 -MJSR0_SX164 -MJSR0_SX254 -MJSR0_SX434 -MJSR0_SX74 -MJWG0_SA2 -MJWG0_SI2155 -MJWG0_SX355 -MJWG0_SX445 -MJWG0_SX85 -MJWS0_SA1 -MJWS0_SA2 -MJWS0_SI1143 -MJWS0_SI1773 -MJWS0_SX243 -MJWS0_SX423 -MJWT0_SA2 -MJWT0_SI751 -MJXA0_SA1 -MJXA0_SA2 -MJXA0_SI1507 -MJXA0_SI2137 -MJXA0_SI877 -MJXA0_SX157 -MJXA0_SX247 -MJXA0_SX337 -MJXA0_SX67 -MJXL0_SA1 -MJXL0_SA2 -MJXL0_SI1795 -MJXL0_SX182 -MJXL0_SX272 -MJXL0_SX362 -MJXL0_SX452 -MJXL0_SX92 -MKAG0_SA2 -MKAG0_SI1609 -MKAG0_SI2239 -MKAG0_SX169 -MKAG0_SX30 -MKAG0_SX439 -MKAG0_SX79 -MKAH0_SA1 -MKAH0_SA2 -MKAH0_SI1528 -MKAH0_SI2158 -MKAH0_SI898 -MKAH0_SX268 -MKAH0_SX358 -MKAH0_SX448 -MKAH0_SX88 -MKAJ0_SA1 -MKAJ0_SI1414 -MKAJ0_SI2044 -MKAJ0_SI784 -MKAJ0_SX244 -MKAJ0_SX334 -MKAJ0_SX424 -MKAJ0_SX64 -MKAM0_SA2 -MKAM0_SI1316 -MKAM0_SX236 -MKAM0_SX416 -MKDB0_SI2132 -MKDB0_SI588 -MKDB0_SI872 -MKDB0_SX242 -MKDB0_SX332 -MKDB0_SX422 -MKDB0_SX62 -MKDD0_SA1 -MKDD0_SX127 -MKDD0_SX217 -MKDD0_SX307 -MKDD0_SX37 -MKDD0_SX397 -MKDT0_SA1 -MKDT0_SA2 -MKDT0_SI2153 -MKDT0_SI893 -MKDT0_SX173 -MKDT0_SX263 -MKDT0_SX353 -MKDT0_SX443 -MKDT0_SX83 -MKES0_SA2 -MKES0_SX263 -MKES0_SX353 -MKES0_SX443 -MKES0_SX83 -MKJO0_SA1 -MKJO0_SA2 -MKJO0_SI2147 -MKJO0_SX167 -MKJO0_SX257 -MKJO0_SX424 -MKJO0_SX77 -MKLN0_SA1 -MKLN0_SA2 -MKLN0_SI1598 -MKLN0_SI2228 -MKLN0_SX158 -MKLN0_SX338 -MKLN0_SX428 -MKLN0_SX68 -MKLR0_SA1 -MKLR0_SI1059 -MKLR0_SI2319 -MKLR0_SX159 -MKLR0_SX249 -MKLR0_SX339 -MKLR0_SX429 -MKLR0_SX69 -MKLS0_SA2 -MKLS0_SI1533 -MKLS0_SX177 -MKLS0_SX267 -MKLS0_SX447 -MKLS1_SI1545 -MKLS1_SI2175 -MKLS1_SX105 -MKLS1_SX15 -MKLS1_SX195 -MKLS1_SX285 -MKLW0_SA2 -MKLW0_SI1844 -MKLW0_SI2201 -MKLW0_SX131 -MKLW0_SX221 -MKLW0_SX401 -MKLW0_SX41 -MKRG0_SA1 -MKRG0_SA2 -MKRG0_SI1491 -MKRG0_SI2121 -MKRG0_SX141 -MKRG0_SX231 -MKRG0_SX31 -MKRG0_SX51 -MKXL0_SA1 -MKXL0_SI1185 -MKXL0_SX105 -MKXL0_SX195 -MKXL0_SX285 -MLBC0_SA2 -MLBC0_SI609 -MLBC0_SX159 -MLBC0_SX339 -MLBC0_SX429 -MLBC0_SX69 -MLEL0_SI1876 -MLEL0_SX346 -MLEL0_SX76 -MLJC0_SA1 -MLJC0_SA2 -MLJC0_SI1855 -MLJC0_SI595 -MLJC0_SX235 -MLJC0_SX325 -MLJC0_SX55 -MLJH0_SI1324 -MLJH0_SX154 -MLJH0_SX334 -MLJH0_SX424 -MLNS0_SA1 -MLNS0_SA2 -MLNS0_SI1407 -MLNS0_SI777 -MLNS0_SX147 -MLNS0_SX237 -MLNS0_SX327 -MLNS0_SX417 -MLNS0_SX57 -MLSH0_SA1 -MLSH0_SA2 -MLSH0_SI2047 -MLSH0_SI787 -MLSH0_SX157 -MLSH0_SX337 -MLSH0_SX427 -MLSH0_SX67 -MMAA0_SI2105 -MMAA0_SX125 -MMAA0_SX215 -MMAA0_SX305 -MMAA0_SX395 -MMAB1_SA1 -MMAB1_SA2 -MMAB1_SI2124 -MMAB1_SX144 -MMAB1_SX414 -MMAB1_SX54 -MMAG0_SI496 -MMAG0_SX226 -MMAG0_SX406 -MMAG0_SX46 -MMAM0_SA1 -MMAM0_SA2 -MMAM0_SI1597 -MMAM0_SI1668 -MMAM0_SX247 -MMAM0_SX337 -MMAM0_SX67 -MMAR0_SA1 -MMAR0_SA2 -MMAR0_SI1336 -MMAR0_SI706 -MMAR0_SX436 -MMAR0_SX76 -MMBS0_SA1 -MMBS0_SA2 -MMBS0_SI1151 -MMBS0_SX251 -MMBS0_SX341 -MMBS0_SX431 -MMBS0_SX71 -MMCC0_SA1 -MMCC0_SI1968 -MMCC0_SI708 -MMCC0_SX168 -MMCC0_SX258 -MMCC0_SX348 -MMCC0_SX438 -MMCC0_SX78 -MMDB0_SA1 -MMDB0_SA2 -MMDB0_SI1358 -MMDB0_SI1617 -MMDB0_SX267 -MMDB0_SX357 -MMDB0_SX447 -MMDB0_SX87 -MMDG0_SI2035 -MMDG0_SX340 -MMDG0_SX430 -MMDG0_SX70 -MMDM0_SA1 -MMDM0_SA2 -MMDM0_SX231 -MMDM0_SX321 -MMDM0_SX411 -MMDM0_SX51 -MMDM1_SA1 -MMDM1_SI1650 -MMDM1_SI783 -MMDM1_SX243 -MMDS0_SA2 -MMDS0_SI1343 -MMDS0_SI1973 -MMDS0_SI713 -MMDS0_SX173 -MMDS0_SX263 -MMDS0_SX353 -MMDS0_SX443 -MMDS0_SX83 -MMEA0_SA2 -MMEA0_SI1388 -MMEA0_SI2018 -MMEA0_SI758 -MMEA0_SX218 -MMEA0_SX308 -MMEA0_SX38 -MMEB0_SA1 -MMEB0_SI1357 -MMEB0_SI1987 -MMEB0_SI727 -MMEB0_SX7 -MMEB0_SX97 -MMGC0_SA1 -MMGC0_SI1935 -MMGC0_SI2184 -MMGC0_SX315 -MMGC0_SX405 -MMGC0_SX45 -MMGG0_SA1 -MMGG0_SA2 -MMGG0_SI1709 -MMGG0_SI2339 -MMGG0_SX179 -MMGG0_SX359 -MMGG0_SX89 -MMGK0_SA1 -MMGK0_SA2 -MMGK0_SI1322 -MMGK0_SI1952 -MMGK0_SI692 -MMGK0_SX152 -MMGK0_SX242 -MMGK0_SX422 -MMJB1_SA1 -MMJB1_SI1408 -MMJB1_SI2038 -MMJB1_SI778 -MMJB1_SX148 -MMJB1_SX238 -MMJB1_SX328 -MMJB1_SX418 -MMJB1_SX58 -MMLM0_SA1 -MMLM0_SA2 -MMLM0_SI1527 -MMLM0_SI897 -MMLM0_SX177 -MMLM0_SX267 -MMLM0_SX357 -MMLM0_SX447 -MMLM0_SX87 -MMPM0_SA1 -MMPM0_SA2 -MMPM0_SI1061 -MMPM0_SI1691 -MMPM0_SI2321 -MMPM0_SX251 -MMPM0_SX341 -MMPM0_SX431 -MMPM0_SX71 -MMRP0_SA1 -MMRP0_SI2034 -MMRP0_SI717 -MMRP0_SI774 -MMRP0_SX234 -MMRP0_SX414 -MMRP0_SX54 -MMSM0_SA1 -MMSM0_SA2 -MMSM0_SI1736 -MMSM0_SX26 -MMSM0_SX296 -MMSM0_SX386 -MMVP0_SI1284 -MMVP0_SI1914 -MMVP0_SX114 -MMVP0_SX204 -MMVP0_SX294 -MMVP0_SX384 -MMWB0_SA2 -MMWB0_SI1619 -MMWB0_SX179 -MMWB0_SX269 -MMWS0_SA1 -MMWS0_SI1518 -MMWS0_SI559 -MMWS0_SI888 -MMWS0_SX258 -MMWS0_SX78 -MMWS1_SA1 -MMWS1_SA2 -MMWS1_SI1071 -MMWS1_SI2331 -MMWS1_SX261 -MMWS1_SX27 -MMWS1_SX351 -MMWS1_SX441 -MMWS1_SX81 -MMXS0_SA1 -MMXS0_SA2 -MMXS0_SI629 -MMXS0_SI876 -MMXS0_SX156 -MMXS0_SX336 -MMXS0_SX66 -MNET0_SA1 -MNET0_SA2 -MNET0_SI1446 -MNET0_SI2076 -MNET0_SX186 -MNET0_SX276 -MNET0_SX366 -MNET0_SX96 -MNTW0_SA1 -MNTW0_SI2328 -MNTW0_SX202 -MNTW0_SX258 -MNTW0_SX348 -MPAR0_SA1 -MPAR0_SA2 -MPAR0_SI1576 -MPAR0_SX226 -MPAR0_SX406 -MPAR0_SX46 -MPEB0_SA1 -MPEB0_SA2 -MPEB0_SX150 -MPEB0_SX420 -MPEB0_SX60 -MPFU0_SA1 -MPFU0_SA2 -MPFU0_SI1888 -MPFU0_SX178 -MPFU0_SX268 -MPFU0_SX358 -MPFU0_SX88 -MPGH0_SA1 -MPGH0_SA2 -MPGH0_SI1554 -MPGH0_SI924 -MPGH0_SX204 -MPGH0_SX294 -MPGH0_SX384 -MPGR0_SA1 -MPGR0_SA2 -MPGR0_SI2040 -MPGR0_SI780 -MPGR0_SX150 -MPGR0_SX420 -MPGR0_SX60 -MPGR1_SA1 -MPGR1_SA2 -MPGR1_SI1269 -MPGR1_SI2129 -MPGR1_SX239 -MPGR1_SX329 -MPGR1_SX419 -MPGR1_SX59 -MPMB0_SX241 -MPPC0_SA2 -MPPC0_SI2042 -MPPC0_SI782 -MPPC0_SX152 -MPPC0_SX242 -MPPC0_SX332 -MPPC0_SX422 -MPPC0_SX62 -MPRB0_SA1 -MPRB0_SA2 -MPRB0_SI1205 -MPRB0_SX125 -MPRB0_SX215 -MPRB0_SX305 -MPRB0_SX35 -MPRB0_SX395 -MPRD0_SA2 -MPRD0_SI1431 -MPRD0_SI2061 -MPRK0_SA2 -MPRK0_SX17 -MPRK0_SX197 -MPRT0_SA2 -MPRT0_SI1210 -MPRT0_SI495 -MPRT0_SI580 -MPRT0_SX130 -MPRT0_SX220 -MPRT0_SX40 -MPRT0_SX400 -MPSW0_SA1 -MPSW0_SA2 -MPSW0_SI1697 -MPSW0_SI2327 -MPSW0_SX24 -MPSW0_SX257 -MPSW0_SX77 -MRAB0_SA1 -MRAB0_SA2 -MRAB0_SI1224 -MRAB0_SI594 -MRAB0_SX144 -MRAB0_SX234 -MRAB0_SX324 -MRAB0_SX414 -MRAB0_SX54 -MRAB1_SA1 -MRAB1_SA2 -MRAB1_SI1478 -MRAB1_SI2108 -MRAB1_SX218 -MRAB1_SX38 -MRAB1_SX398 -MRAI0_SI1954 -MRAI0_SX162 -MRAI0_SX252 -MRAI0_SX342 -MRAM0_SI1275 -MRAM0_SI1905 -MRAM0_SX105 -MRAM0_SX195 -MRAM0_SX285 -MRAM0_SX375 -MRAV0_SA1 -MRAV0_SA2 -MRAV0_SI1008 -MRAV0_SI1638 -MRAV0_SI2268 -MRAV0_SX108 -MRAV0_SX18 -MRAV0_SX198 -MRAV0_SX288 -MRAV0_SX378 -MRBC0_SA1 -MRBC0_SA2 -MRBC0_SI1665 -MRBC0_SI599 -MRBC0_SX149 -MRBC0_SX239 -MRBC0_SX59 -MRCG0_SA1 -MRCG0_SI2058 -MRCG0_SX258 -MRCG0_SX78 -MRCW0_SA2 -MRCW0_SI1371 -MRCW0_SI2001 -MRCW0_SX111 -MRCW0_SX201 -MRCW0_SX21 -MRCW0_SX381 -MRDD0_SA1 -MRDD0_SA2 -MRDD0_SI1050 -MRDD0_SI2310 -MRDD0_SX240 -MRDD0_SX330 -MRDM0_SA1 -MRDM0_SA2 -MRDM0_SI965 -MRDM0_SX155 -MRDM0_SX245 -MRDM0_SX425 -MRDS0_SA2 -MRDS0_SI1167 -MRDS0_SI1797 -MRDS0_SI537 -MRDS0_SX177 -MRDS0_SX267 -MRDS0_SX357 -MRDS0_SX447 -MRDS0_SX87 -MREE0_SA1 -MREE0_SA2 -MREE0_SI1734 -MREE0_SX114 -MREE0_SX204 -MREE0_SX294 -MREE0_SX384 -MREH1_SA2 -MREH1_SI2229 -MREH1_SX159 -MREH1_SX339 -MREH1_SX429 -MREM0_SA1 -MREM0_SI1591 -MREM0_SI961 -MREM0_SX151 -MREM0_SX241 -MREM0_SX331 -MREM0_SX421 -MREM0_SX61 -MREW1_SA1 -MREW1_SA2 -MREW1_SI1500 -MREW1_SI2130 -MREW1_SX150 -MREW1_SX240 -MREW1_SX330 -MREW1_SX420 -MREW1_SX60 -MRFK0_SA1 -MRFK0_SA2 -MRFK0_SI1706 -MRFK0_SI2336 -MRFK0_SX176 -MRFK0_SX266 -MRFK0_SX356 -MRFK0_SX86 -MRFL0_SA2 -MRFL0_SI1786 -MRFL0_SX346 -MRGM0_SA1 -MRGM0_SI1162 -MRGM0_SI1792 -MRGM0_SX416 -MRGM0_SX82 -MRGS0_SA1 -MRGS0_SI1986 -MRGS0_SX276 -MRGS0_SX366 -MRGS0_SX96 -MRHL0_SA1 -MRHL0_SA2 -MRHL0_SI1515 -MRHL0_SI2145 -MRHL0_SX165 -MRHL0_SX255 -MRHL0_SX75 -MRJB1_SI1020 -MRJB1_SX300 -MRJH0_SA1 -MRJH0_SI914 -MRJH0_SX259 -MRJH0_SX439 -MRJM0_SA1 -MRJM0_SA2 -MRJM0_SI1095 -MRJM0_SI1228 -MRJM0_SI1858 -MRJM0_SX238 -MRJM0_SX328 -MRJM0_SX418 -MRJM0_SX58 -MRJM1_SA1 -MRJM1_SI668 -MRJM1_SX218 -MRJM1_SX308 -MRJM1_SX38 -MRJM1_SX398 -MRJT0_SA1 -MRJT0_SI1805 -MRJT0_SX148 -MRJT0_SX238 -MRKM0_SA1 -MRKM0_SX187 -MRKM0_SX277 -MRKM0_SX7 -MRKM0_SX97 -MRLD0_SA1 -MRLD0_SI1594 -MRLD0_SI964 -MRLD0_SX244 -MRLD0_SX334 -MRLD0_SX64 -MRLJ0_SA2 -MRLJ0_SI1420 -MRLJ0_SI2050 -MRLJ0_SX160 -MRLJ0_SX430 -MRLJ0_SX70 -MRLJ1_SI1671 -MRLJ1_SI2332 -MRLJ1_SX141 -MRLJ1_SX231 -MRLJ1_SX411 -MRLJ1_SX51 -MRLK0_SA1 -MRLK0_SA2 -MRLK0_SI2140 -MRLK0_SX303 -MRLK0_SX33 -MRLK0_SX393 -MRLR0_SA1 -MRLR0_SA2 -MRLR0_SI1826 -MRLR0_SI566 -MRLR0_SX116 -MRLR0_SX206 -MRLR0_SX26 -MRLR0_SX296 -MRLR0_SX386 -MRMB0_SA1 -MRMB0_SI2211 -MRMB0_SI951 -MRMB0_SX141 -MRMB0_SX231 -MRMB0_SX321 -MRMB0_SX51 -MRMG0_SA2 -MRMG0_SI1710 -MRMG0_SI2340 -MRMG0_SX180 -MRMG0_SX270 -MRMG0_SX360 -MRMG0_SX90 -MRMH0_SA1 -MRMH0_SA2 -MRMH0_SI1021 -MRMH0_SX211 -MRMH0_SX301 -MRMH0_SX31 -MRMH0_SX391 -MRML0_SI2051 -MRML0_SI791 -MRML0_SX431 -MRML0_SX71 -MRMS0_SA1 -MRMS0_SA2 -MRMS0_SI1113 -MRMS0_SI2100 -MRMS0_SX120 -MRMS0_SX210 -MRMS0_SX30 -MRMS0_SX300 -MRMS0_SX390 -MRPC1_SA1 -MRPC1_SA2 -MRPC1_SI1482 -MRPC1_SI2026 -MRPC1_SX132 -MRPC1_SX222 -MRPC1_SX312 -MRPC1_SX402 -MRPC1_SX42 -MRRE0_SI704 -MRRE0_SX254 -MRRE0_SX434 -MRSO0_SA1 -MRSO0_SA2 -MRSO0_SI1659 -MRSO0_SI2289 -MRSO0_SX219 -MRSO0_SX309 -MRSO0_SX399 -MRSP0_SA1 -MRSP0_SA2 -MRSP0_SI2059 -MRSP0_SI799 -MRSP0_SX169 -MRSP0_SX196 -MRSP0_SX439 -MRSP0_SX79 -MRTC0_SA1 -MRTC0_SA2 -MRTC0_SI2088 -MRTC0_SI828 -MRTC0_SX108 -MRTC0_SX18 -MRTC0_SX198 -MRTC0_SX288 -MRTJ0_SA2 -MRTJ0_SI1551 -MRTJ0_SI2032 -MRTJ0_SX322 -MRTJ0_SX412 -MRVG0_SA1 -MRVG0_SA2 -MRVG0_SI1770 -MRVG0_SI510 -MRVG0_SX150 -MRVG0_SX330 -MRVG0_SX420 -MRVG0_SX60 -MRWA0_SA1 -MRWA0_SA2 -MRWA0_SI1603 -MRWA0_SI2233 -MRWA0_SX253 -MRWA0_SX343 -MRWA0_SX433 -MRWS0_SA1 -MRWS0_SA2 -MRWS0_SX112 -MRWS0_SX202 -MRWS0_SX292 -MRXB0_SA1 -MRXB0_SI1585 -MRXB0_SX145 -MRXB0_SX235 -MRXB0_SX325 -MRXB0_SX55 -MSAH1_SA1 -MSAH1_SA2 -MSAH1_SI1049 -MSAH1_SI2309 -MSAH1_SX149 -MSAH1_SX239 -MSAH1_SX329 -MSAH1_SX419 -MSAH1_SX59 -MSAS0_SA1 -MSAS0_SA2 -MSAS0_SI2006 -MSAS0_SX26 -MSAS0_SX296 -MSAT0_SA2 -MSAT0_SI1526 -MSAT0_SI2156 -MSAT0_SI896 -MSAT0_SX176 -MSAT0_SX266 -MSAT0_SX356 -MSAT0_SX446 -MSAT0_SX86 -MSAT1_SA1 -MSAT1_SA2 -MSAT1_SI1073 -MSAT1_SI1703 -MSAT1_SI2333 -MSAT1_SX173 -MSAT1_SX353 -MSDB0_SA1 -MSDB0_SA2 -MSDB0_SI1007 -MSDB0_SI1637 -MSDB0_SI2267 -MSDB0_SX107 -MSDB0_SX17 -MSDH0_SA1 -MSDH0_SA2 -MSDH0_SI2113 -MSDH0_SX260 -MSDH0_SX350 -MSDS0_SA2 -MSDS0_SI1707 -MSDS0_SI2337 -MSDS0_SX177 -MSDS0_SX447 -MSDS0_SX87 -MSEM1_SA1 -MSEM1_SA2 -MSEM1_SX360 -MSEM1_SX450 -MSEM1_SX90 -MSES0_SA1 -MSES0_SA2 -MSES0_SI2216 -MSES0_SI2219 -MSES0_SX149 -MSES0_SX329 -MSES0_SX59 -MSFH0_SA2 -MSFH0_SI1216 -MSFH0_SI586 -MSFH0_SX226 -MSFH0_SX46 -MSFV0_SA1 -MSFV0_SA2 -MSFV0_SI1262 -MSFV0_SX182 -MSFV0_SX272 -MSFV0_SX452 -MSJK0_SA1 -MSJK0_SA2 -MSJK0_SI2226 -MSJK0_SI966 -MSJK0_SX156 -MSJK0_SX246 -MSJK0_SX426 -MSJK0_SX66 -MSMC0_SA1 -MSMC0_SA2 -MSMC0_SI1907 -MSMC0_SI647 -MSMC0_SX107 -MSMC0_SX17 -MSMC0_SX197 -MSMC0_SX287 -MSMC0_SX377 -MSMR0_SA1 -MSMR0_SA2 -MSMR0_SI1405 -MSMR0_SI775 -MSMR0_SX145 -MSMR0_SX235 -MSMR0_SX325 -MSMR0_SX55 -MSMS0_SA2 -MSMS0_SI2063 -MSMS0_SI803 -MSMS0_SX263 -MSMS0_SX353 -MSMS0_SX443 -MSRG0_SA2 -MSRG0_SI1851 -MSRG0_SI591 -MSRG0_SX141 -MSRG0_SX231 -MSRG0_SX321 -MSRG0_SX411 -MSRG0_SX51 -MSRR0_SA1 -MSRR0_SA2 -MSRR0_SI1131 -MSRR0_SX141 -MSRR0_SX231 -MSRR0_SX30 -MSRR0_SX411 -MSRR0_SX51 -MSTF0_SA1 -MSTF0_SA2 -MSTF0_SI1396 -MSTF0_SX136 -MSTF0_SX226 -MSTF0_SX406 -MSVS0_SA1 -MSVS0_SI1568 -MSVS0_SX128 -MSVS0_SX218 -MSVS0_SX38 -MTAB0_SA1 -MTAB0_SA2 -MTAB0_SI2202 -MTAB0_SI942 -MTAB0_SX132 -MTAB0_SX222 -MTAB0_SX402 -MTAB0_SX42 -MTAS0_SA1 -MTAS0_SA2 -MTAS0_SI1385 -MTAS0_SI2015 -MTAS0_SI755 -MTAS0_SX125 -MTAS0_SX305 -MTAT0_SA2 -MTAT0_SI1740 -MTAT0_SX120 -MTAT0_SX210 -MTAT0_SX30 -MTAT0_SX300 -MTAT1_SA1 -MTAT1_SA2 -MTAT1_SI1409 -MTAT1_SI1627 -MTAT1_SX239 -MTAT1_SX419 -MTBC0_SA1 -MTBC0_SA2 -MTBC0_SI1173 -MTBC0_SX183 -MTBC0_SX273 -MTBC0_SX347 -MTBC0_SX363 -MTBC0_SX93 -MTCS0_SA1 -MTCS0_SI1972 -MTCS0_SX172 -MTCS0_SX262 -MTCS0_SX352 -MTCS0_SX442 -MTDB0_SA1 -MTDB0_SA2 -MTDB0_SI2031 -MTDB0_SX141 -MTDB0_SX231 -MTDB0_SX321 -MTDB0_SX411 -MTDB0_SX51 -MTDP0_SI1274 -MTDP0_SI2151 -MTDP0_SX261 -MTDP0_SX441 -MTDP0_SX81 -MTER0_SI527 -MTER0_SX167 -MTER0_SX17 -MTER0_SX257 -MTER0_SX77 -MTJG0_SA2 -MTJG0_SI1520 -MTJG0_SI890 -MTJG0_SX350 -MTJG0_SX440 -MTJG0_SX80 -MTJM0_SA1 -MTJM0_SA2 -MTJM0_SI1226 -MTJM0_SI655 -MTJM0_SX236 -MTJM0_SX326 -MTJM0_SX416 -MTJM0_SX56 -MTJS0_SA1 -MTJS0_SI1192 -MTJS0_SX112 -MTJS0_SX202 -MTJS0_SX22 -MTJS0_SX292 -MTJU0_SA1 -MTJU0_SA2 -MTJU0_SI2269 -MTJU0_SI760 -MTJU0_SX220 -MTJU0_SX310 -MTJU0_SX40 -MTKD0_SA1 -MTKD0_SA2 -MTKD0_SI1187 -MTKD0_SI1817 -MTKD0_SX17 -MTKD0_SX197 -MTKD0_SX377 -MTKP0_SA1 -MTKP0_SA2 -MTKP0_SX123 -MTKP0_SX213 -MTKP0_SX303 -MTKP0_SX33 -MTKP0_SX393 -MTLB0_SA2 -MTLB0_SI1764 -MTLB0_SI504 -MTLB0_SX144 -MTLB0_SX414 -MTLB0_SX54 -MTLC0_SA2 -MTLC0_SI847 -MTLC0_SX127 -MTLC0_SX217 -MTLC0_SX307 -MTLC0_SX37 -MTLC0_SX397 -MTML0_SA1 -MTML0_SA2 -MTML0_SI1065 -MTML0_SI1695 -MTML0_SX255 -MTML0_SX345 -MTML0_SX75 -MTMN0_SA1 -MTMN0_SX164 -MTMN0_SX254 -MTMN0_SX344 -MTMN0_SX74 -MTMT0_SA1 -MTMT0_SI1118 -MTMT0_SX128 -MTMT0_SX218 -MTMT0_SX308 -MTMT0_SX38 -MTMT0_SX398 -MTPF0_SA1 -MTPF0_SA2 -MTPF0_SI1235 -MTPF0_SI1865 -MTPF0_SI605 -MTPF0_SX155 -MTPF0_SX245 -MTPF0_SX335 -MTPF0_SX425 -MTPG0_SA1 -MTPG0_SA2 -MTPG0_SI2013 -MTPG0_SX123 -MTPG0_SX213 -MTPG0_SX33 -MTPG0_SX393 -MTPP0_SA1 -MTPP0_SA2 -MTPP0_SI2138 -MTPP0_SI878 -MTPP0_SX158 -MTPP0_SX248 -MTPP0_SX428 -MTPP0_SX68 -MTPR0_SA1 -MTPR0_SA2 -MTPR0_SI1600 -MTPR0_SI506 -MTPR0_SX250 -MTPR0_SX70 -MTQC0_SA2 -MTQC0_SI2071 -MTQC0_SX271 -MTQC0_SX361 -MTRC0_SA1 -MTRC0_SA2 -MTRC0_SI1623 -MTRC0_SI993 -MTRC0_SX170 -MTRC0_SX183 -MTRC0_SX273 -MTRC0_SX363 -MTRC0_SX93 -MTRR0_SA1 -MTRR0_SA2 -MTRR0_SI1548 -MTRR0_SI2178 -MTRR0_SX108 -MTRR0_SX18 -MTRR0_SX378 -MTRT0_SA1 -MTRT0_SI1857 -MTRT0_SI597 -MTRT0_SX147 -MTRT0_SX237 -MTRT0_SX417 -MTWH1_SA1 -MTWH1_SA2 -MTWH1_SI1512 -MTWH1_SI2142 -MTWH1_SI882 -MTWH1_SX162 -MTWH1_SX252 -MTWH1_SX342 -MTWH1_SX432 -MTXS0_SI1690 -MTXS0_SX250 -MTXS0_SX340 -MTXS0_SX70 -MVJH0_SA1 -MVJH0_SA2 -MVJH0_SI2186 -MVJH0_SX116 -MVJH0_SX26 -MVJH0_SX386 -MVLO0_SA2 -MVLO0_SI1147 -MVLO0_SI1777 -MVLO0_SX157 -MVLO0_SX247 -MVLO0_SX337 -MVLO0_SX427 -MVLO0_SX67 -MVRW0_SA1 -MVRW0_SI1485 -MVRW0_SI2115 -MVRW0_SI855 -MVRW0_SX315 -MVRW0_SX405 -MVRW0_SX45 -MWAC0_SA1 -MWAC0_SI2231 -MWAC0_SI971 -MWAC0_SX71 -MWAD0_SA1 -MWAD0_SA2 -MWAD0_SI1062 -MWAD0_SI1749 -MWAD0_SI2322 -MWAD0_SX162 -MWAD0_SX252 -MWAD0_SX342 -MWAR0_SA2 -MWAR0_SI2305 -MWAR0_SX145 -MWAR0_SX235 -MWAR0_SX325 -MWAR0_SX415 -MWAR0_SX55 -MWCH0_SA1 -MWCH0_SA2 -MWCH0_SI1622 -MWCH0_SX272 -MWCH0_SX362 -MWCH0_SX92 -MWDK0_SX266 -MWDK0_SX356 -MWDK0_SX446 -MWEM0_SA1 -MWEM0_SI1950 -MWEM0_SX240 -MWEM0_SX330 -MWEM0_SX60 -MWGR0_SA1 -MWGR0_SA2 -MWGR0_SI1606 -MWGR0_SI2236 -MWGR0_SI976 -MWGR0_SX166 -MWGR0_SX256 -MWGR0_SX436 -MWGR0_SX76 -MWRE0_SA1 -MWRE0_SI1687 -MWRE0_SI2317 -MWRE0_SX157 -MWRP0_SA2 -MWRP0_SI1525 -MWRP0_SI2073 -MWRP0_SX183 -MWRP0_SX3 -MWRP0_SX93 -MWSB0_SA1 -MWSB0_SA2 -MWSB0_SI1626 -MWSB0_SI2256 -MWSB0_SX186 -MWSB0_SX366 -MWSB0_SX6 -MWSB0_SX96 -MWSH0_SA1 -MWSH0_SA2 -MWSH0_SI2266 -MWSH0_SX346 -MWSH0_SX436 -MZMB0_SA2 -MZMB0_SI1166 -MZMB0_SI1796 -MZMB0_SI536 -MZMB0_SX176 -MZMB0_SX266 -MZMB0_SX356 -MZMB0_SX446 -MZMB0_SX86 diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/config/timit_unmatched/train_text.uid b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/config/timit_unmatched/train_text.uid deleted file mode 100644 index 0e0c2517c9415ce76d5863781f621402cd15b911..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/config/timit_unmatched/train_text.uid +++ /dev/null @@ -1,1000 +0,0 @@ -FAEM0_SI762 -FAEM0_SX42 -FAJW0_SA1 -FAJW0_SX3 -FAJW0_SX93 -FALK0_SX186 -FALK0_SX6 -FALR0_SI1325 -FBAS0_SA1 -FBAS0_SX217 -FBCG1_SA1 -FBCG1_SX172 -FBCG1_SX442 -FBCH0_SX236 -FBCH0_SX416 -FBLV0_SA1 -FBLV0_SI1058 -FBLV0_SX338 -FBLV0_SX68 -FBMH0_SA1 -FBMJ0_SI815 -FCAG0_SA1 -FCAG0_SX153 -FCAG0_SX243 -FCAJ0_SI1479 -FCAJ0_SX309 -FCDR1_SX106 -FCDR1_SX196 -FCEG0_SA2 -FCJF0_SA1 -FCJF0_SX127 -FCJS0_SI1607 -FCJS0_SI2237 -FCJS0_SX257 -FCKE0_SA2 -FCKE0_SX121 -FCLT0_SI2068 -FCLT0_SX448 -FCLT0_SX88 -FCMG0_SA2 -FCMG0_SI1872 -FCMG0_SX72 -FCMM0_SA1 -FCMM0_SA2 -FCMM0_SX183 -FCRZ0_SI2053 -FCRZ0_SX433 -FCYL0_SA1 -FCYL0_SX37 -FDAS1_SI2091 -FDAS1_SX201 -FDAS1_SX381 -FDAW0_SI1406 -FDFB0_SA1 -FDFB0_SA2 -FDFB0_SI2010 -FDFB0_SX58 -FDJH0_SX305 -FDML0_SA2 -FDML0_SX159 -FDML0_SX249 -FDML0_SX429 -FDMY0_SA2 -FDMY0_SX27 -FDNC0_SX198 -FDNC0_SX288 -FDTD0_SX211 -FDXW0_SA1 -FDXW0_SX251 -FDXW0_SX341 -FDXW0_SX71 -FEAC0_SX165 -FEAC0_SX75 -FEAR0_SI622 -FECD0_SX68 -FEEH0_SA1 -FEEH0_SI1742 -FEEH0_SI471 -FEEH0_SX122 -FEME0_SA1 -FEME0_SX155 -FEME0_SX65 -FETB0_SA1 -FETB0_SI1148 -FETB0_SX158 -FEXM0_SI1101 -FGCS0_SX136 -FGCS0_SX226 -FGCS0_SX316 -FGCS0_SX406 -FGDP0_SA1 -FGMB0_SI1775 -FGMB0_SX245 -FHLM0_SX390 -FHXS0_SA2 -FHXS0_SX445 -FJDM2_SA1 -FJDM2_SX232 -FJDM2_SX52 -FJHK0_SX302 -FJKL0_SX212 -FJKL0_SX392 -FJLG0_SI2306 -FJLR0_SA1 -FJRP1_SI2062 -FJRP1_SX82 -FJSK0_SA1 -FJSP0_SX264 -FJSP0_SX354 -FJSP0_SX444 -FJWB1_SA1 -FJWB1_SX345 -FJWB1_SX435 -FJXM0_SA1 -FJXM0_SI581 -FJXM0_SX401 -FJXP0_SA1 -FJXP0_SI1122 -FJXP0_SX132 -FKAA0_SX128 -FKAA0_SX398 -FKDE0_SA1 -FKDE0_SX151 -FKDE0_SX241 -FKDE0_SX421 -FKDE0_SX61 -FKDW0_SX397 -FKFB0_SA2 -FKFB0_SX348 -FKFB0_SX78 -FKKH0_SA1 -FKKH0_SA2 -FKKH0_SX120 -FKKH0_SX390 -FKLC0_SX355 -FKLC1_SI2308 -FKLC1_SX238 -FKLC1_SX328 -FKLC1_SX418 -FKLH0_SA2 -FKLH0_SX177 -FKSR0_SA1 -FKSR0_SA2 -FKSR0_SI1747 -FKSR0_SI487 -FKSR0_SX217 -FLAC0_SX451 -FLAG0_SA2 -FLAG0_SX114 -FLAG0_SX204 -FLAG0_SX24 -FLAG0_SX384 -FLEH0_SI1681 -FLEH0_SI2311 -FLEH0_SX331 -FLET0_SA1 -FLHD0_SI1827 -FLHD0_SX354 -FLJA0_SA1 -FLJA0_SI2338 -FLJD0_SI886 -FLJD0_SX76 -FLJG0_SA2 -FLKM0_SA2 -FLKM0_SI686 -FLKM0_SX260 -FLKM0_SX80 -FLMA0_SA1 -FLMA0_SI613 -FLMA0_SX433 -FLMA0_SX73 -FLMC0_SX22 -FLMK0_SI1035 -FLMK0_SX315 -FLMK0_SX405 -FLOD0_SI1917 -FLOD0_SX117 -FLOD0_SX171 -FLOD0_SX297 -FLTM0_SA1 -FLTM0_SI1070 -FLTM0_SI2330 -FMAH1_SA2 -FMAH1_SX159 -FMBG0_SA2 -FMBG0_SI2264 -FMEM0_SI747 -FMEM0_SX387 -FMJB0_SI547 -FMJB0_SX97 -FMJF0_SA2 -FMJU0_SX309 -FMJU0_SX399 -FMKC0_SI1702 -FMKC0_SX442 -FMKC0_SX82 -FMKF0_SX186 -FMPG0_SA2 -FNKL0_SI1522 -FNTB0_SI1203 -FNTB0_SI573 -FNTB0_SX303 -FPAB1_SI1471 -FPAB1_SX211 -FPAC0_SA2 -FPAD0_SA2 -FPAD0_SX356 -FPAD0_SX86 -FPAF0_SA2 -FPAF0_SX154 -FPAZ0_SA1 -FPAZ0_SA2 -FPAZ0_SX243 -FPJF0_SA1 -FPJF0_SX146 -FPJF0_SX56 -FPLS0_SI1590 -FPLS0_SX330 -FPMY0_SA1 -FPMY0_SX343 -FREH0_SA1 -FREH0_SA2 -FREH0_SX415 -FRJB0_SX347 -FRLL0_SX434 -FSAG0_SA1 -FSAG0_SX243 -FSAH0_SA1 -FSAH0_SA2 -FSAH0_SX164 -FSAH0_SX434 -FSBK0_SA2 -FSBK0_SI1069 -FSBK0_SX169 -FSCN0_SA2 -FSCN0_SI626 -FSCN0_SX266 -FSCN0_SX446 -FSCN0_SX86 -FSDC0_SA2 -FSDC0_SX142 -FSDC0_SX322 -FSDC0_SX52 -FSDJ0_SI485 -FSDJ0_SX215 -FSDJ0_SX305 -FSDJ0_SX395 -FSGF0_SX117 -FSJG0_SX130 -FSJK1_SA2 -FSJK1_SX125 -FSJK1_SX35 -FSJS0_SX181 -FSJW0_SI1963 -FSJW0_SX433 -FSKC0_SI1416 -FSKC0_SI786 -FSKC0_SX246 -FSKL0_SI1529 -FSKL0_SX449 -FSKP0_SA2 -FSLS0_SX156 -FSLS0_SX426 -FSMA0_SA2 -FSMA0_SX181 -FSMM0_SX144 -FSMM0_SX234 -FSMS1_SX244 -FSMS1_SX347 -FSPM0_SA2 -FSPM0_SX161 -FSPM0_SX71 -FSRH0_SI1931 -FSRH0_SI671 -FSRH0_SX221 -FSRH0_SX401 -FTAJ0_SI699 -FTAJ0_SX159 -FTAJ0_SX249 -FTAJ0_SX429 -FTBR0_SX21 -FTBW0_SA1 -FTMG0_SI1532 -FTMG0_SI2162 -FTMG0_SX452 -FVFB0_SA2 -FVFB0_SX132 -FVFB0_SX42 -FVKB0_SA1 -FVMH0_SA2 -FVMH0_SX116 -FVMH0_SX26 -MABC0_SI1620 -MABC0_SI2041 -MABC0_SI781 -MADC0_SX107 -MADC0_SX377 -MADD0_SA2 -MADD0_SI1295 -MADD0_SX178 -MADD0_SX268 -MADD0_SX88 -MAEB0_SX450 -MAEO0_SA1 -MAFM0_SI939 -MAFM0_SX129 -MAFM0_SX309 -MAJP0_SA2 -MAKB0_SI1646 -MAKB0_SX26 -MAKB0_SX386 -MAKR0_SX362 -MAKR0_SX92 -MAPV0_SX213 -MARC0_SA2 -MARC0_SX108 -MARC0_SX18 -MARC0_SX198 -MARW0_SI1906 -MBAR0_SA1 -MBAR0_SX419 -MBAR0_SX59 -MBBR0_SI2315 -MBBR0_SX65 -MBCG0_SA1 -MBCG0_SI486 -MBEF0_SI1281 -MBEF0_SI1911 -MBEF0_SI651 -MBEF0_SX21 -MBEF0_SX381 -MBGT0_SA2 -MBGT0_SX261 -MBGT0_SX351 -MBGT0_SX441 -MBJV0_SA1 -MBJV0_SI617 -MBJV0_SX347 -MBMA0_SI592 -MBMA0_SX232 -MBMA0_SX52 -MBMA1_SI2214 -MBMA1_SX54 -MBML0_SA2 -MBML0_SI1169 -MBML0_SX89 -MBOM0_SA2 -MBOM0_SI2274 -MBOM0_SX294 -MBSB0_SA1 -MBSB0_SX3 -MBTH0_SA2 -MBTH0_SX122 -MBTH0_SX32 -MCAE0_SX277 -MCAL0_SA2 -MCAL0_SI1768 -MCDC0_SA1 -MCDC0_SX212 -MCDD0_SA2 -MCDD0_SI883 -MCDD0_SX253 -MCDD0_SX433 -MCDR0_SI1154 -MCEF0_SX235 -MCEF0_SX415 -MCEW0_SA2 -MCHL0_SX87 -MCLK0_SX310 -MCLM0_SA1 -MCLM0_SI2086 -MCLM0_SI826 -MCPM0_SA1 -MCPM0_SX114 -MCPM0_SX294 -MCPM0_SX384 -MCSS0_SI750 -MCTH0_SA1 -MCTH0_SX39 -MCXM0_SX91 -MDAC0_SA1 -MDAC0_SX181 -MDAC0_SX361 -MDAS0_SX6 -MDBB1_SX106 -MDBB1_SX16 -MDBB1_SX376 -MDBP0_SX168 -MDCD0_SI1415 -MDCD0_SX245 -MDCD0_SX425 -MDCM0_SX40 -MDCM0_SX400 -MDDC0_SI2049 -MDDC0_SI789 -MDDC0_SX159 -MDDC0_SX69 -MDED0_SA1 -MDED0_SA2 -MDEF0_SX123 -MDEF0_SX303 -MDHL0_SI1439 -MDHL0_SX269 -MDHL0_SX449 -MDHS0_SA1 -MDHS0_SA2 -MDHS0_SI1530 -MDHS0_SI2160 -MDJM0_SX105 -MDJM0_SX15 -MDKS0_SX436 -MDLB0_SA2 -MDLC0_SX405 -MDLC1_SA2 -MDLC1_SI2065 -MDLC1_SI2144 -MDLC1_SX445 -MDLC2_SI2244 -MDLC2_SX354 -MDLH0_SA2 -MDLM0_SI1234 -MDLM0_SI1864 -MDLM0_SX154 -MDLM0_SX424 -MDLR0_SA1 -MDLR0_SA2 -MDLR0_SI1863 -MDLR0_SI603 -MDLR0_SX153 -MDLR1_SA1 -MDLR1_SA2 -MDMA0_SI1430 -MDMA0_SX260 -MDMA0_SX80 -MDMT0_SA1 -MDMT0_SA2 -MDMT0_SI1832 -MDMT0_SX122 -MDMT0_SX32 -MDNS0_SA2 -MDNS0_SI2271 -MDNS0_SX201 -MDNS0_SX21 -MDPB0_SX416 -MDPK0_SI1053 -MDPK0_SX333 -MDPK0_SX423 -MDPS0_SI719 -MDPS0_SX359 -MDRD0_SA1 -MDRD0_SX32 -MDSJ0_SI2092 -MDSS0_SA2 -MDSS0_SX441 -MDSS1_SA1 -MDSS1_SI1327 -MDSS1_SI697 -MDSS1_SX157 -MDSS1_SX67 -MDTB0_SI1200 -MDTB0_SI1830 -MDTB0_SX120 -MDWD0_SA2 -MDWD0_SX270 -MDWD0_SX90 -MDWH0_SX215 -MDWH0_SX305 -MDWM0_SA1 -MDWM0_SA2 -MDWM0_SX16 -MDWM0_SX286 -MEAL0_SA2 -MEAL0_SI2177 -MEAL0_SX107 -MEAL0_SX347 -MEDR0_SA1 -MEDR0_SA2 -MEDR0_SI1374 -MEFG0_SA1 -MEGJ0_SA2 -MEGJ0_SX257 -MEGJ0_SX3 -MEJL0_SA1 -MEJL0_SX152 -MEJL0_SX242 -MEJS0_SI610 -MEJS0_SX160 -MEJS0_SX340 -MESG0_SX432 -MESJ0_SX187 -MESJ0_SX97 -MEWM0_SI718 -MEWM0_SX178 -MEWM0_SX88 -MFER0_SI862 -MFER0_SX142 -MFRM0_SX345 -MFRM0_SX435 -MFWK0_SI1879 -MFWK0_SX169 -MFXS0_SX54 -MFXV0_SA2 -MFXV0_SX105 -MGAF0_SA1 -MGAF0_SX22 -MGAF0_SX382 -MGAG0_SA2 -MGAK0_SX226 -MGAK0_SX46 -MGAR0_SX132 -MGAW0_SI535 -MGAW0_SX175 -MGES0_SA1 -MGES0_SI2111 -MGES0_SI851 -MGJC0_SA2 -MGJC0_SX75 -MGRL0_SI2127 -MGRL0_SI867 -MGRL0_SX147 -MGRP0_SA2 -MGSH0_SA2 -MGSH0_SI1806 -MGSH0_SX127 -MGSH0_SX276 -MGSH0_SX6 -MGSL0_SA1 -MGSL0_SI534 -MGSL0_SX264 -MGXP0_SX187 -MGXP0_SX7 -MHBS0_SX315 -MHBS0_SX45 -MHIT0_SA1 -MHJB0_SA1 -MHJB0_SI1017 -MHMG0_SX195 -MHMR0_SA1 -MHMR0_SI489 -MHRM0_SA1 -MHRM0_SI958 -MHRM0_SX148 -MHRM0_SX58 -MHXL0_SI1772 -MHXL0_SX242 -MILB0_SA2 -MJAC0_SX307 -MJAC0_SX71 -MJAE0_SX174 -MJAI0_SA1 -MJAI0_SA2 -MJBG0_SX62 -MJDA0_SI1031 -MJDA0_SX311 -MJDE0_SI463 -MJDG0_SA2 -MJDG0_SI1042 -MJDG0_SI1705 -MJDM0_SA1 -MJDM0_SI974 -MJEB0_SI656 -MJEB0_SX296 -MJEB1_SA2 -MJEB1_SX207 -MJEB1_SX387 -MJEE0_SA1 -MJEE0_SX247 -MJEE0_SX337 -MJFH0_SA2 -MJFH0_SI1107 -MJFR0_SX75 -MJHI0_SA1 -MJHI0_SX158 -MJJB0_SA1 -MJJB0_SX239 -MJJJ0_SX443 -MJJM0_SA2 -MJJM0_SI827 -MJJM0_SX107 -MJKR0_SA1 -MJKR0_SI571 -MJLB0_SX176 -MJLG1_SX292 -MJLS0_SX106 -MJMA0_SA1 -MJMA0_SA2 -MJMD0_SA2 -MJMD0_SX308 -MJMD0_SX38 -MJMM0_SX85 -MJPG0_SI1191 -MJPG0_SX111 -MJPG0_SX201 -MJPG0_SX21 -MJPM0_SA2 -MJPM0_SX378 -MJPM1_SI2280 -MJPM1_SX401 -MJRA0_SA1 -MJRA0_SA2 -MJRA0_SI1236 -MJRA0_SI1866 -MJRA0_SX426 -MJRG0_SI1366 -MJRG0_SI1996 -MJRG0_SX376 -MJRH0_SX225 -MJRH1_SA1 -MJRH1_SI514 -MJRH1_SX154 -MJRH1_SX244 -MJRH1_SX424 -MJRK0_SA1 -MJRK0_SA2 -MJRK0_SI1662 -MJRK0_SX160 -MJRK0_SX250 -MJRK0_SX430 -MJRP0_SA1 -MJRP0_SA2 -MJRP0_SX225 -MJSR0_SA1 -MJSR0_SI1424 -MJSR0_SX344 -MJWG0_SA1 -MJWG0_SX265 -MJWS0_SI513 -MJWS0_SX153 -MJWS0_SX63 -MJWT0_SA1 -MJWT0_SX121 -MJWT0_SX211 -MJWT0_SX301 -MJWT0_SX31 -MJWT0_SX391 -MJXA0_SX427 -MJXL0_SI542 -MKAG0_SA1 -MKAG0_SX259 -MKAJ0_SA2 -MKAJ0_SX154 -MKAM0_SA1 -MKAM0_SX146 -MKAM0_SX326 -MKAM0_SX56 -MKDB0_SA1 -MKDB0_SA2 -MKDB0_SX152 -MKDD0_SA2 -MKES0_SA1 -MKES0_SI1253 -MKES0_SI1883 -MKES0_SX173 -MKJO0_SI1517 -MKJO0_SI887 -MKJO0_SX437 -MKLN0_SI968 -MKLN0_SX248 -MKLR0_SA2 -MKLR0_SI1689 -MKLS0_SA1 -MKLS0_SX357 -MKLS0_SX87 -MKLS1_SA1 -MKLS1_SA2 -MKLS1_SX375 -MKLW0_SA1 -MKRG0_SX411 -MKXL0_SA2 -MKXL0_SX15 -MKXL0_SX375 -MLBC0_SA1 -MLBC0_SI1869 -MLBC0_SX249 -MLEL0_SA1 -MLEL0_SA2 -MLEL0_SI1246 -MLEL0_SX256 -MLEL0_SX436 -MLJC0_SX145 -MLJC0_SX415 -MLJH0_SX64 -MLNS0_SI2037 -MMAA0_SA1 -MMAA0_SA2 -MMAA0_SX35 -MMAB1_SI1494 -MMAB1_SX234 -MMAG0_SA2 -MMAG0_SI1126 -MMAG0_SX316 -MMAM0_SI2227 -MMAM0_SX157 -MMAM0_SX427 -MMAR0_SX256 -MMBS0_SI1781 -MMCC0_SA2 -MMDB0_SX177 -MMDG0_SA1 -MMDG0_SA2 -MMDG0_SI520 -MMDG0_SX160 -MMDG0_SX250 -MMDM0_SI1941 -MMDM0_SI681 -MMDM0_SX141 -MMDM1_SA2 -MMDM1_SI2043 -MMDM1_SX423 -MMDM1_SX63 -MMDS0_SA1 -MMEA0_SA1 -MMEA0_SX128 -MMEA0_SX398 -MMEB0_SA2 -MMEB0_SX187 -MMEB0_SX367 -MMGC0_SA2 -MMGC0_SX135 -MMGC0_SX225 -MMGG0_SX269 -MMGK0_SX332 -MMGK0_SX62 -MMJB1_SA2 -MMRP0_SA2 -MMRP0_SX144 -MMSM0_SX116 -MMSM0_SX206 -MMVP0_SA1 -MMVP0_SA2 -MMWB0_SI989 -MMWB0_SX89 -MMWS0_SA2 -MMWS0_SX168 -MMWS0_SX348 -MMWS0_SX438 -MMWS1_SI1701 -MMXS0_SI2136 -MMXS0_SX246 -MMXS0_SX426 -MNET0_SI816 -MNET0_SX6 -MNTW0_SA2 -MNTW0_SX168 -MNTW0_SX78 -MPAR0_SI2206 -MPAR0_SI946 -MPAR0_SX136 -MPAR0_SX316 -MPEB0_SI1034 -MPEB0_SI1860 -MPEB0_SX240 -MPEB0_SX330 -MPFU0_SI628 -MPFU0_SX448 -MPGH0_SX114 -MPGH0_SX24 -MPGR0_SX240 -MPGR0_SX330 -MPGR1_SX149 -MPPC0_SA1 -MPRD0_SA1 -MPRD0_SX261 -MPRD0_SX351 -MPRD0_SX441 -MPRD0_SX81 -MPRK0_SI1727 -MPRK0_SX107 -MPRK0_SX377 -MPRT0_SA1 -MPRT0_SX310 -MPSW0_SI1067 -MPSW0_SX167 -MPSW0_SX437 -MRAB1_SX128 -MRAB1_SX308 -MRAI0_SA1 -MRAI0_SA2 -MRAI0_SX72 -MRAM0_SA1 -MRAM0_SA2 -MRAM0_SX15 -MRBC0_SI1859 -MRBC0_SX329 -MRBC0_SX419 -MRCG0_SI798 -MRCG0_SX168 -MRCW0_SA1 -MRCW0_SX291 -MRDD0_SI1680 -MRDD0_SX150 -MRDD0_SX277 -MRDD0_SX60 -MRDM0_SI1595 -MRDM0_SX65 -MRDS0_SA1 -MREE0_SX24 -MREH1_SX249 -MREH1_SX69 -MREM0_SA2 -MREW1_SI870 -MRFK0_SX446 -MRFL0_SA1 -MRFL0_SX256 -MRFL0_SX436 -MRFL0_SX76 -MRGM0_SA2 -MRGM0_SX262 -MRGS0_SA2 -MRGS0_SX186 -MRHL0_SI885 -MRHL0_SX345 -MRHL0_SX435 -MRJB1_SA1 -MRJB1_SA2 -MRJB1_SX210 -MRJB1_SX30 -MRJB1_SX390 -MRJH0_SA2 -MRJH0_SX307 -MRJH0_SX79 -MRJM0_SX148 -MRJM1_SA2 -MRJM1_SI1298 -MRJM1_SI1928 -MRJM1_SX128 -MRJT0_SA2 -MRJT0_SI1498 -MRJT0_SX328 -MRJT0_SX418 -MRKM0_SA2 -MRKM0_SX367 -MRLD0_SA2 -MRLD0_SI2224 -MRLD0_SX154 -MRLD0_SX424 -MRLJ0_SA1 -MRLJ0_SX250 -MRLJ0_SX340 -MRLJ1_SA1 -MRLJ1_SA2 -MRLJ1_SX321 -MRLK0_SI843 -MRLK0_SX123 -MRLK0_SX213 -MRMB0_SA2 -MRMB0_SI1581 -MRMB0_SX411 -MRMG0_SA1 -MRMG0_SI1080 -MRMG0_SX450 -MRMH0_SI1349 -MRMH0_SI2281 -MRMH0_SX121 -MRML0_SA2 -MRML0_SX341 -MRPC1_SI2112 -MRRE0_SA2 -MRRE0_SX164 -MRRE0_SX344 -MRRE0_SX74 -MRSO0_SX129 -MRSO0_SX39 -MRSP0_SX259 -MRTC0_SX378 -MRVG0_SI1140 -MRVG0_SX240 -MRWA0_SI973 -MRWA0_SX163 -MRWA0_SX73 -MRWS0_SI1732 -MRWS0_SI472 -MRWS0_SX22 -MRWS0_SX382 -MRXB0_SA2 -MRXB0_SX415 -MSAH1_SI1679 -MSAS0_SX116 -MSAS0_SX206 -MSAS0_SX386 -MSAT0_SA1 -MSAT1_SX263 -MSAT1_SX443 -MSAT1_SX83 -MSDB0_SX197 -MSDB0_SX287 -MSDB0_SX377 -MSDH0_SI2240 -MSDH0_SX440 -MSDH0_SX80 -MSDS0_SA1 -MSEM1_SI1440 -MSEM1_SX180 -MSEM1_SX270 -MSES0_SI1589 -MSES0_SX239 -MSES0_SX419 -MSFH0_SX316 -MSFV0_SI1892 -MSFV0_SX362 -MSFV0_SX92 -MSMR0_SX415 -MSMS0_SA1 -MSMS0_SX173 -MSMS0_SX83 -MSRG0_SA1 -MSRG0_SI1221 -MSTF0_SI766 -MSTF0_SX316 -MSTF0_SX46 -MSVS0_SA2 -MSVS0_SX308 -MTAS0_SX215 -MTAS0_SX35 -MTAS0_SX395 -MTAT0_SX390 -MTAT1_SX59 -MTBC0_SI1803 -MTCS0_SA2 -MTCS0_SI2265 -MTCS0_SX82 -MTDP0_SA2 -MTER0_SA2 -MTER0_SI1787 -MTJG0_SA1 -MTJG0_SI2157 -MTJG0_SX260 -MTJM0_SI1856 -MTJM0_SX146 -MTJU0_SX130 -MTJU0_SX400 -MTKD0_SX107 -MTKD0_SX287 -MTKP0_SI1023 -MTLB0_SA1 -MTLB0_SX234 -MTLC0_SA1 -MTML0_SI2325 -MTML0_SX165 -MTMN0_SA2 -MTMN0_SI1064 -MTMN0_SI2324 -MTMN0_SX434 -MTMT0_SA2 -MTMT0_SI1748 -MTPF0_SX65 -MTPG0_SI1383 -MTPG0_SI753 -MTPG0_SX303 -MTPP0_SX338 -MTPR0_SX340 -MTQC0_SI480 -MTQC0_SX91 -MTRR0_SX198 -MTRR0_SX288 -MTRT0_SA2 -MTRT0_SX254 -MTRT0_SX57 -MTWH1_SX72 -MTXS0_SA1 -MTXS0_SA2 -MVJH0_SI926 -MVJH0_SX206 -MVJH0_SX296 -MVLO0_SA1 -MVRW0_SA2 -MVRW0_SX135 -MVRW0_SX225 -MWAC0_SA2 -MWAC0_SX341 -MWAC0_SX431 -MWAD0_SX432 -MWAD0_SX72 -MWAR0_SA1 -MWAR0_SI1675 -MWCH0_SI1895 -MWCH0_SI2252 -MWCH0_SX182 -MWCH0_SX452 -MWDK0_SA1 -MWDK0_SA2 -MWDK0_SI2017 -MWDK0_SI806 -MWDK0_SX176 -MWDK0_SX86 -MWEM0_SA2 -MWEM0_SI1320 -MWEM0_SI1393 -MWEM0_SX150 -MWGR0_SX346 -MWRE0_SX247 -MWRE0_SX337 -MWRE0_SX427 -MWRP0_SA1 -MWRP0_SX273 -MWRP0_SX363 -MWSB0_SX276 -MWSH0_SX256 -MWSH0_SX76 -MZMB0_SA1 diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/config/timit_unmatched/valid.uid b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/config/timit_unmatched/valid.uid deleted file mode 100644 index e99edfe937854a5f47a2f0384f0e067487336883..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/config/timit_unmatched/valid.uid +++ /dev/null @@ -1,620 +0,0 @@ -FAEM0_SI1392 -FAJW0_SI1263 -FAJW0_SI633 -FALK0_SI658 -FALR0_SX335 -FAPB0_SI1063 -FAPB0_SI2323 -FAPB0_SX433 -FBAS0_SI1472 -FBAS0_SI2066 -FBCG1_SX352 -FBCH0_SI959 -FBJL0_SI922 -FBLV0_SI1688 -FBMH0_SI1136 -FBMH0_SI970 -FBMJ0_SA1 -FBMJ0_SI1776 -FBMJ0_SI516 -FBMJ0_SX336 -FCDR1_SI1186 -FCDR1_SI1816 -FCDR1_SI556 -FCDR1_SX286 -FCKE0_SI1741 -FCKE0_SI481 -FCLT0_SI808 -FCMG0_SI1142 -FCMG0_SX432 -FCMM0_SI1957 -FCMM0_SX420 -FCYL0_SI667 -FCYL0_SX349 -FDAS1_SI1461 -FDAS1_SI831 -FDAW0_SI1271 -FDAW0_SI2036 -FDJH0_SI935 -FDKN0_SI1202 -FDKN0_SX181 -FDKN0_SX451 -FDMY0_SA1 -FDMY0_SI567 -FDMY0_SI714 -FDMY0_SX387 -FDNC0_SI1278 -FDNC0_SI1908 -FDTD0_SA1 -FDTD0_SX321 -FEAC0_SI615 -FEAR0_SX352 -FECD0_SA1 -FECD0_SI1418 -FECD0_SI788 -FEME0_SI875 -FEME0_SX335 -FEXM0_SA1 -FEXM0_SI482 -FEXM0_SX366 -FGDP0_SI988 -FGDP0_SX88 -FGMB0_SI1145 -FGMB0_SX335 -FGRW0_SA1 -FGRW0_SI1152 -FGRW0_SX162 -FGRW0_SX432 -FHLM0_SX120 -FHLM0_SX349 -FHXS0_SA1 -FHXS0_SI1075 -FHXS0_SI2302 -FHXS0_SX175 -FJDM2_SA2 -FJDM2_SX142 -FJEN0_SA1 -FJEN0_SX327 -FJEN0_SX417 -FJHK0_SI2282 -FJKL0_SI932 -FJLG0_SI1889 -FJLR0_SI1231 -FJRB0_SX402 -FJRP1_SA1 -FJRP1_SI1432 -FJRP1_SX262 -FJRP1_SX352 -FJSK0_SI1052 -FJSP0_SI1434 -FJWB1_SI748 -FJXM0_SX311 -FJXM0_SX41 -FJXP0_SI1752 -FKAA0_SA1 -FKDE0_SI1141 -FKDE0_SI1771 -FKDW0_SI1207 -FKDW0_SI1891 -FKFB0_SI1608 -FKFB0_SX438 -FKKH0_SI1290 -FKKH0_SI1920 -FKLC0_SI985 -FKLC0_SX175 -FKLC1_SI1048 -FKLH0_SI1257 -FKSR0_SX366 -FLAC0_SI1339 -FLAG0_SI1464 -FLAG0_SI834 -FLEH0_SI1051 -FLET0_SI507 -FLJA0_SI1078 -FLJA0_SX178 -FLJD0_SI1516 -FLJG0_SI981 -FLJG0_SX171 -FLJG0_SX351 -FLKM0_SA1 -FLKM0_SI620 -FLKM0_SX350 -FLKM0_SX440 -FLMC0_SI1372 -FLMK0_SA1 -FLMK0_SI1229 -FLTM0_SX170 -FLTM0_SX350 -FLTM0_SX440 -FMAH1_SI879 -FMBG0_SI1160 -FMEM0_SA1 -FMEM0_SX333 -FMJB0_SI1177 -FMJF0_SI624 -FMJF0_SX174 -FMJF0_SX84 -FMJU0_SI1389 -FMKC0_SI1041 -FMKF0_SI1018 -FMPG0_SA1 -FMPG0_SI972 -FMPG0_SX162 -FMPG0_SX342 -FMPG0_SX432 -FNKL0_SI892 -FNTB0_SI679 -FPAB1_SA1 -FPAB1_SI2101 -FPAB1_SI841 -FPAC0_SI1921 -FPAC0_SI661 -FPAD0_SI716 -FPAD0_SX176 -FPAF0_SA1 -FPAF0_SI1054 -FPAZ0_SI2223 -FPAZ0_SI963 -FPJF0_SI1259 -FPJF0_SX352 -FPLS0_SI960 -FPMY0_SI1153 -FPMY0_SI523 -FREH0_SI1945 -FRLL0_SI805 -FSAG0_SI1323 -FSAG0_SX153 -FSAG0_SX333 -FSAG0_SX423 -FSAH0_SI614 -FSAH0_SX327 -FSAK0_SI1300 -FSBK0_SX349 -FSCN0_SA1 -FSCN0_SI705 -FSCN0_SX176 -FSDC0_SI1312 -FSDJ0_SI1115 -FSGF0_SI2187 -FSGF0_SI927 -FSJG0_SA1 -FSJG0_SA2 -FSJG0_SI940 -FSJG0_SX220 -FSJG0_SX40 -FSJG0_SX400 -FSJS0_SA1 -FSJS0_SX451 -FSJW0_SI1333 -FSKP0_SI1098 -FSMA0_SI991 -FSMA0_SX451 -FSMM0_SX324 -FSPM0_SI1241 -FSPM0_SX251 -FSRH0_SX311 -FSSB0_SI1712 -FSSB0_SX362 -FTBR0_SI1402 -FTBR0_SI921 -FTBW0_SI715 -FTBW0_SX175 -FTLG0_SI1743 -FTLG0_SI483 -FTMG0_SI902 -FVFB0_SI1510 -FVKB0_SX349 -FVMH0_SI1466 -FVMH0_SI836 -MADC0_SI1367 -MADC0_SI737 -MAEB0_SI1411 -MAEO0_SI1326 -MAJP0_SI1704 -MAJP0_SX174 -MAKB0_SA2 -MAKB0_SI1016 -MAKB0_SI2276 -MAKB0_SX116 -MAPV0_SI1293 -MAPV0_SI663 -MARW0_SX286 -MARW0_SX349 -MBBR0_SI1055 -MBBR0_SX335 -MBCG0_SI957 -MBCG0_SX327 -MBGT0_SI1841 -MBGT0_SX171 -MBMA0_SI1222 -MBMA1_SI954 -MBMA1_SX324 -MBTH0_SI2102 -MBWP0_SX349 -MCAE0_SI1447 -MCAE0_SI2077 -MCAE0_SI817 -MCAL0_SI1138 -MCDR0_SI1784 -MCDR0_SI524 -MCEF0_SI842 -MCEW0_SA1 -MCEW0_SI2072 -MCEW0_SI812 -MCEW0_SX362 -MCEW0_SX452 -MCHL0_SI1347 -MCHL0_SI1404 -MCLK0_SI2290 -MCLK0_SI650 -MCPM0_SI1824 -MCSS0_SI1380 -MCSS0_SI688 -MCTM0_SI1350 -MCTM0_SI1980 -MDAC0_SI631 -MDAS0_SI1896 -MDAS0_SI636 -MDBP0_SI528 -MDBP0_SX438 -MDCD0_SI785 -MDCD0_SX335 -MDCM0_SI1480 -MDDC0_SI1419 -MDED0_SI540 -MDEF0_SI1123 -MDEM0_SA1 -MDEM0_SI608 -MDEM0_SI800 -MDEM0_SX428 -MDHS0_SI900 -MDJM0_SI1455 -MDKS0_SX166 -MDKS0_SX346 -MDLB0_SI1306 -MDLB0_SX136 -MDLB0_SX406 -MDLC0_SI1395 -MDLC0_SI2025 -MDLC1_SI1435 -MDLH0_SX160 -MDLH0_SX430 -MDLM0_SI604 -MDLR0_SX333 -MDLR1_SI669 -MDMA0_SX170 -MDMA0_SX350 -MDMA0_SX440 -MDNS0_SI1011 -MDNS0_SI873 -MDPB0_SI1760 -MDPB0_SI866 -MDRD0_SI752 -MDSJ0_SI1462 -MDSJ0_SX438 -MDWD0_SI1260 -MDWH0_SA1 -MDWH0_SI1168 -MDWH0_SI665 -MDWM0_SI916 -MEDR0_SI2004 -MEFG0_SI491 -MEFG0_SI598 -MEGJ0_SA1 -MEGJ0_SI1337 -MEGJ0_SI707 -MEGJ0_SX167 -MEJS0_SI1240 -MESG0_SI702 -MESJ0_SI2039 -MFWK0_SX349 -MFXS0_SX324 -MFXV0_SI1005 -MFXV0_SI1342 -MGAF0_SI1282 -MGAG0_SI691 -MGAK0_SI1036 -MGAK0_SX136 -MGAR0_SX312 -MGAW0_SI1165 -MGES0_SX311 -MGJC0_SX435 -MGRL0_SX327 -MGRP0_SI1317 -MGRP0_SX327 -MGSH0_SI1176 -MGSH0_SI546 -MGSL0_SI797 -MGXP0_SI1087 -MGXP0_SI525 -MHBS0_SI945 -MHIT0_SI983 -MHMG0_SI735 -MHMR0_SI1692 -MILB0_SI903 -MJAC0_SI701 -MJAC0_SX251 -MJAE0_SX84 -MJAI0_SI682 -MJAI0_SI710 -MJDC0_SI531 -MJDE0_SA1 -MJDE0_SI1120 -MJDE0_SI490 -MJDE0_SX220 -MJDM0_SI1340 -MJDM0_SX170 -MJDM0_SX350 -MJEB0_SX170 -MJEB1_SI1467 -MJEB1_SI837 -MJFR0_SA1 -MJFR0_SX435 -MJHI0_SI1328 -MJJJ0_SI1163 -MJJM0_SI1251 -MJLB0_SI1616 -MJLS0_SI1726 -MJMA0_SI2125 -MJMD0_SI2288 -MJMM0_SI1255 -MJMM0_SX175 -MJPG0_SI1821 -MJPM0_SI1368 -MJPM1_SX311 -MJRA0_SX336 -MJRG0_SI736 -MJRG0_SX352 -MJRH0_SI1840 -MJRH1_SI1558 -MJRK0_SI880 -MJRP0_SI1845 -MJSR0_SI2054 -MJSR0_SI794 -MJWG0_SI813 -MJWG0_SI895 -MJWG0_SX175 -MJWS0_SX333 -MJWT0_SI1291 -MJWT0_SI1381 -MJXL0_SI1172 -MKAG0_SI979 -MKAH0_SX178 -MKAM0_SI1250 -MKAM0_SI1465 -MKDD0_SI1567 -MKDD0_SI2197 -MKDD0_SI937 -MKDT0_SI814 -MKES0_SI623 -MKLS0_SI1437 -MKLS0_SI2067 -MKLS1_SI915 -MKLW0_SI1571 -MKLW0_SX311 -MKRG0_SI861 -MKXL0_SI1815 -MKXL0_SI1958 -MLBC0_SI1239 -MLEL0_SI616 -MLEL0_SX166 -MLJC0_SI1225 -MLJH0_SA1 -MLJH0_SA2 -MLJH0_SI1422 -MLJH0_SI694 -MLJH0_SX244 -MLSH0_SI1417 -MLSH0_SX247 -MMAA0_SI1588 -MMAA0_SI845 -MMAB1_SI864 -MMAB1_SX324 -MMAG0_SA1 -MMAG0_SI1756 -MMAG0_SX136 -MMAR0_SI1966 -MMAR0_SX166 -MMAR0_SX346 -MMBS0_SI521 -MMBS0_SX161 -MMCC0_SI1338 -MMDB0_SI987 -MMDG0_SI1780 -MMDM0_SI1311 -MMDM1_SX153 -MMDM1_SX333 -MMEB0_SX327 -MMGC0_SI1305 -MMGG0_SI1079 -MMGG0_SX449 -MMLM0_SI2150 -MMPM0_SX161 -MMRP0_SX324 -MMSM0_SI1106 -MMSM0_SI476 -MMVP0_SI654 -MMVP0_SX347 -MMWB0_SA1 -MMWB0_SI2249 -MMWB0_SX359 -MMWB0_SX449 -MNTW0_SI1068 -MNTW0_SI1698 -MPEB0_SI600 -MPFU0_SI1258 -MPGH0_SI675 -MPGR0_SI1410 -MPGR1_SI1499 -MPMB0_SA1 -MPMB0_SA2 -MPMB0_SI1501 -MPMB0_SI2131 -MPMB0_SI871 -MPMB0_SX151 -MPMB0_SX331 -MPMB0_SX421 -MPMB0_SX61 -MPPC0_SI1412 -MPRB0_SI1215 -MPRB0_SI575 -MPRD0_SI801 -MPRD0_SX171 -MPRK0_SA1 -MPRK0_SI1097 -MPRK0_SI467 -MPRK0_SX287 -MRAB0_SI1854 -MRAB1_SI848 -MRAI0_SI2052 -MRAI0_SI792 -MRAI0_SX432 -MRAM0_SI1951 -MRCG0_SA2 -MRCG0_SI1428 -MRCG0_SX348 -MRCG0_SX438 -MRCW0_SI741 -MRDM0_SI1044 -MRDM0_SX335 -MREE0_SI1104 -MREE0_SI1959 -MREH1_SA1 -MREH1_SI1599 -MREH1_SI969 -MREM0_SI511 -MRFK0_SI1076 -MRFL0_SI1156 -MRFL0_SI526 -MRFL0_SX166 -MRGM0_SI532 -MRGM0_SX172 -MRGM0_SX442 -MRGS0_SI1356 -MRGS0_SI726 -MRGS0_SX6 -MRJB1_SI1413 -MRJB1_SI2021 -MRJB1_SX120 -MRJH0_SI1519 -MRJH0_SI889 -MRJH0_SX169 -MRJT0_SI868 -MRJT0_SX58 -MRKM0_SI1267 -MRKM0_SI1391 -MRKM0_SI637 -MRLJ0_SI790 -MRLJ1_SI2301 -MRLK0_SI1468 -MRLR0_SI1196 -MRML0_SA1 -MRML0_SI1421 -MRML0_SX161 -MRML0_SX251 -MRMS0_SI2057 -MRRE0_SA1 -MRRE0_SI1334 -MRRE0_SI952 -MRSO0_SI1206 -MRSP0_SI1429 -MRTC0_SI1458 -MRTJ0_SA1 -MRTJ0_SI772 -MRTJ0_SX142 -MRTJ0_SX232 -MRTJ0_SX52 -MRWS0_SI1102 -MRXB0_SI2215 -MRXB0_SI955 -MSAS0_SI1376 -MSAS0_SI746 -MSDH0_SI980 -MSDH0_SX170 -MSDS0_SI1077 -MSDS0_SX267 -MSDS0_SX357 -MSEM1_SI2070 -MSEM1_SI810 -MSFH0_SA1 -MSFH0_SI1738 -MSFH0_SX136 -MSFH0_SX406 -MSFV0_SI632 -MSJK0_SI1596 -MSJK0_SX336 -MSMC0_SI509 -MSMR0_SI1150 -MSMS0_SI1433 -MSRR0_SI1761 -MSRR0_SI501 -MSTF0_SI852 -MSVS0_SI2198 -MSVS0_SI938 -MSVS0_SX398 -MTAB0_SI1572 -MTAB0_SX312 -MTAT0_SA1 -MTAT0_SI1110 -MTAT0_SI811 -MTAT1_SI779 -MTAT1_SX149 -MTAT1_SX329 -MTBC0_SI543 -MTCS0_SI712 -MTDB0_SI1401 -MTDB0_SI771 -MTDP0_SA1 -MTDP0_SI1521 -MTDP0_SX171 -MTDP0_SX351 -MTER0_SA1 -MTER0_SI1157 -MTER0_SX437 -MTJG0_SX170 -MTJS0_SA2 -MTJS0_SI1822 -MTJS0_SI562 -MTJS0_SX382 -MTJU0_SI2020 -MTKD0_SI630 -MTKP0_SI2283 -MTKP0_SI454 -MTLB0_SI1134 -MTLB0_SX324 -MTLC0_SI1313 -MTLC0_SI1477 -MTML0_SX435 -MTMN0_SI582 -MTMT0_SI488 -MTPP0_SI1508 -MTPR0_SI2230 -MTPR0_SX160 -MTPR0_SX430 -MTQC0_SA1 -MTQC0_SI1441 -MTQC0_SX181 -MTQC0_SX451 -MTRC0_SI589 -MTRR0_SI918 -MTRT0_SI1227 -MTXS0_SI1060 -MTXS0_SI2320 -MTXS0_SX160 -MTXS0_SX430 -MVJH0_SI1556 -MVLO0_SI517 -MWAC0_SI1601 -MWAC0_SX161 -MWAC0_SX251 -MWAR0_SI1045 -MWDK0_SI1436 -MWEM0_SX420 -MWRE0_SA2 -MWRE0_SI1057 -MWRE0_SX67 -MWRP0_SI1443 -MWSB0_SI996 -MWSH0_SI1426 -MWSH0_SI796 -MWSH0_SX166 diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/data/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/data/__init__.py deleted file mode 100644 index b7c91873aaf1e324ab5309093e86ef905ae6d79f..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/data/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from .extracted_features_dataset import ExtractedFeaturesDataset -from .random_input_dataset import RandomInputDataset - -__all__ = [ - "ExtractedFeaturesDataset", - "RandomInputDataset", -] diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/data/extracted_features_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/data/extracted_features_dataset.py deleted file mode 100644 index 71750856dff0f1aa18c65948ba8f51baf23d4e47..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/data/extracted_features_dataset.py +++ /dev/null @@ -1,142 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - - -import contextlib -import logging -import os - -import numpy as np -import torch -from fairseq.data import FairseqDataset, data_utils - -logger = logging.getLogger(__name__) - - -class ExtractedFeaturesDataset(FairseqDataset): - def __init__( - self, - path, - split, - min_length=3, - max_length=None, - labels=None, - label_dict=None, - shuffle=True, - sort_by_length=True, - ): - super().__init__() - - self.min_length = min_length - self.max_length = max_length - self.shuffle = shuffle - self.sort_by_length = sort_by_length - self.label_dict = label_dict - - if labels is not None: - assert label_dict is not None - - self.sizes = [] - self.offsets = [] - self.labels = [] - - path = os.path.join(path, split) - data_path = path - self.data = np.load(data_path + ".npy", mmap_mode="r") - - offset = 0 - skipped = 0 - - if not os.path.exists(path + f".{labels}"): - labels = None - - with open(data_path + ".lengths", "r") as len_f, open( - path + f".{labels}", "r" - ) if labels is not None else contextlib.ExitStack() as lbl_f: - for line in len_f: - length = int(line.rstrip()) - lbl = None if labels is None else next(lbl_f).rstrip().split() - if length >= min_length and ( - max_length is None or length <= max_length - ): - self.sizes.append(length) - self.offsets.append(offset) - if lbl is not None: - self.labels.append(lbl) - offset += length - - self.sizes = np.asarray(self.sizes) - self.offsets = np.asarray(self.offsets) - - logger.info(f"loaded {len(self.offsets)}, skipped {skipped} samples") - - def __getitem__(self, index): - offset = self.offsets[index] - end = self.sizes[index] + offset - feats = torch.from_numpy(self.data[offset:end].copy()).float() - - res = {"id": index, "features": feats} - if len(self.labels) > 0: - res["target"] = self.label_dict.encode_line( - self.labels[index], - line_tokenizer=lambda x: x, - append_eos=False, - ) - - return res - - def __len__(self): - return len(self.sizes) - - def collater(self, samples): - if len(samples) == 0: - return {} - - features = [s["features"] for s in samples] - sizes = [len(s) for s in features] - - target_size = max(sizes) - - collated_features = features[0].new_zeros( - len(features), target_size, features[0].size(-1) - ) - padding_mask = torch.BoolTensor(collated_features.shape[:-1]).fill_(False) - for i, (f, size) in enumerate(zip(features, sizes)): - collated_features[i, :size] = f - padding_mask[i, size:] = True - - res = { - "id": torch.LongTensor([s["id"] for s in samples]), - "net_input": {"features": collated_features, "padding_mask": padding_mask}, - } - - if len(self.labels) > 0: - target = data_utils.collate_tokens( - [s["target"] for s in samples], - pad_idx=self.label_dict.pad(), - left_pad=False, - ) - res["target"] = target - return res - - def num_tokens(self, index): - return self.size(index) - - def size(self, index): - return self.sizes[index] - - def ordered_indices(self): - """Return an ordered list of indices. Batches will be constructed based - on this order.""" - if self.shuffle: - order = [np.random.permutation(len(self))] - else: - order = [np.arange(len(self))] - - if self.sort_by_length: - order.append(self.sizes) - return np.lexsort(order)[::-1] - else: - return order[0] diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/data/random_input_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/data/random_input_dataset.py deleted file mode 100644 index 886505616cc7f7a515ecebf34fae5c2bc541de03..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/data/random_input_dataset.py +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import random -from typing import List - -from fairseq.data import BaseWrapperDataset, data_utils - - -class RandomInputDataset(BaseWrapperDataset): - def __init__( - self, - dataset, - random_input_dataset, - input_key_path: List[str], - add_to_input, - pad_idx, - ): - super().__init__(dataset) - self.random_input_dataset = random_input_dataset - if isinstance(input_key_path, str): - input_key_path = [input_key_path] - assert len(input_key_path) > 0 - self.input_key_path = input_key_path - self.add_to_input = add_to_input - self.pad_idx = pad_idx - - def get_target(self, item): - target_loc = item - for p in self.input_key_path[:-1]: - target_loc = target_loc[p] - return self.input_key_path[-1], target_loc - - def get_target_value(self, item): - k, target_loc = self.get_target(item) - return target_loc[k] - - def __getitem__(self, index): - item = self.dataset[index] - k, target_loc = self.get_target(item) - target_loc[k] = random.choice(self.random_input_dataset) - return item - - def collater(self, samples): - collated = self.dataset.collater(samples) - if len(collated) == 0: - return collated - indices = set(collated["id"].tolist()) - - random_inputs = data_utils.collate_tokens( - [self.get_target_value(s) for s in samples if s["id"] in indices], - pad_idx=self.pad_idx, - left_pad=False, - ) - k, target_loc = self.get_target( - collated if not self.add_to_input else collated["net_input"] - ) - target_loc[k] = random_inputs - - return collated diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/README.md b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/README.md deleted file mode 100644 index 314984fcbb6825169193b21bd6bb3fca5fd2503b..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/README.md +++ /dev/null @@ -1,56 +0,0 @@ -# Self-Training with Kaldi HMM Models -This folder contains recipes for self-training on pseudo phone transcripts and -decoding into phones or words with [kaldi](https://github.com/kaldi-asr/kaldi). - -To start, download and install kaldi follow its instruction, and place this -folder in `path/to/kaldi/egs`. - -## Training -Assuming the following has been prepared: -- `w2v_dir`: contains features `{train,valid}.{npy,lengths}`, real transcripts `{train,valid}.${label}`, and dict `dict.${label}.txt` -- `lab_dir`: contains pseudo labels `{train,valid}.txt` -- `arpa_lm`: Arpa-format n-gram phone LM for decoding -- `arpa_lm_bin`: Arpa-format n-gram phone LM for unsupervised model selection to be used with KenLM - -Set these variables in `train.sh`, as well as `out_dir`, the output directory, -and then run it. - -The output will be: -``` -==== WER w.r.t. real transcript (select based on unsupervised metric) -INFO:root:./out/exp/mono/decode_valid/scoring/14.0.0.tra.txt: score 0.9178 wer 28.71% lm_ppl 24.4500 gt_wer 25.57% -INFO:root:./out/exp/tri1/decode_valid/scoring/17.1.0.tra.txt: score 0.9257 wer 26.99% lm_ppl 30.8494 gt_wer 21.90% -INFO:root:./out/exp/tri2b/decode_valid/scoring/8.0.0.tra.txt: score 0.7506 wer 23.15% lm_ppl 25.5944 gt_wer 15.78% -``` -where `wer` is the word eror rate with respect to the pseudo label, `gt_wer` to -the ground truth label, `lm_ppl` the language model perplexity of HMM prediced -transcripts, and `score` is the unsupervised metric for model selection. We -choose the model and the LM parameter of the one with the lowest score. In the -example above, it is `tri2b`, `8.0.0`. - - -## Decoding into Phones -In `decode_phone.sh`, set `out_dir` the same as used in `train.sh`, set -`dec_exp` and `dec_lmparam` to the selected model and LM parameter (e.g. -`tri2b` and `8.0.0` in the above example). `dec_script` needs to be set -according to `dec_exp`: for mono/tri1/tri2b, use `decode.sh`; for tri3b, use -`decode_fmllr.sh`. - -The output will be saved at `out_dir/dec_data` - - -## Decoding into Words -`decode_word_step1.sh` prepares WFSTs for word decoding. Besides the variables -mentioned above, set -- `wrd_arpa_lm`: Arpa-format n-gram word LM for decoding -- `wrd_arpa_lm_bin`: Arpa-format n-gram word LM for unsupervised model selection - -`decode_word_step1.sh` decodes the `train` and `valid` split into word and runs -unsupervised model selection using the `valid` split. The output is like: -``` -INFO:root:./out/exp/tri2b/decodeword_valid/scoring/17.0.0.tra.txt: score 1.8693 wer 24.97% lm_ppl 1785.5333 gt_wer 31.45% -``` - -After determining the LM parameter (`17.0.0` in the example above), set it in -`decode_word_step2.sh` and run it. The output will be saved at -`out_dir/dec_data_word`. diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/cmd.sh b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/cmd.sh deleted file mode 100644 index e74953194d41f0d93855d41b2acef08556d92477..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/cmd.sh +++ /dev/null @@ -1,15 +0,0 @@ -# you can change cmd.sh depending on what type of queue you are using. -# If you have no queueing system and want to run on a local machine, you -# can change all instances 'queue.pl' to run.pl (but be careful and run -# commands one by one: most recipes will exhaust the memory on your -# machine). queue.pl works with GridEngine (qsub). slurm.pl works -# with slurm. Different queues are configured differently, with different -# queue names and different ways of specifying things like memory; -# to account for these differences you can create and edit the file -# conf/queue.conf to match your queue's configuration. Search for -# conf/queue.conf in http://kaldi-asr.org/doc/queue.html for more information, -# or search for the string 'default_config' in utils/queue.pl or utils/slurm.pl. - -export train_cmd="run.pl --mem 2G" -export decode_cmd="run.pl --mem 4G" -export mkgraph_cmd="run.pl --mem 8G" diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/decode_phone.sh b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/decode_phone.sh deleted file mode 100644 index 947342a0b7d8f50bcf4164b284ef3303a1247b64..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/decode_phone.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash - -# decode into phones (and prepare a new data directory for HMM outputs) - -. ./path.sh - -set -eu - -out_dir= # same as in train.sh -dec_lmparam= # LM hyperparameters (e.g., 7.0.0) -dec_exp= -dec_script= -dec_splits="train valid" -dec_data_dir=$out_dir/dec_data # where to write HMM output - -data_dir=${out_dir}/data - -local/decode.sh --nj 40 --graph_name graph \ - --val_sets "$dec_splits" --decode_script $dec_script \ - $out_dir/exp/$dec_exp $data_dir $data_dir/lang_test - -if [ ! -z $dec_lmparam ]; then - for x in $dec_splits; do - mkdir -p $dec_data_dir/$x - cp $data_dir/$x/{feats.scp,cmvn.scp,utt2spk,spk2utt} $dec_data_dir/$x/ - - tra=$out_dir/exp/$dec_exp/decode_${x}/scoring/${dec_lmparam}.tra - cat $tra | utils/int2sym.pl -f 2- $data_dir/lang/words.txt | \ - sed 's:<UNK>::g' | sed 's:<SIL>::g' > $dec_data_dir/${x}/text - utils/fix_data_dir.sh $dec_data_dir/${x} - echo "WER on ${x} is" $(compute-wer ark:$data_dir/${x}_gt/text ark:$dec_data_dir/$x/text | cut -d" " -f2-) - done -fi diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/decode_word_step1.sh b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/decode_word_step1.sh deleted file mode 100644 index c1276bbe4d0e02deb984c7c10d6c0486dce09a5f..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/decode_word_step1.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash - -# prepare word WFSTs, reference data, and decode - -set -eu - -w2v_dir= # same as in train.sh -out_dir= # same as in train.sh -lexicon= # word to phone mapping -wrd_arpa_lm= # word LM -wrd_arpa_lm_bin= # word LM for KenLM, used in unsupervised selection - -dec_exp= # what HMM stage to decode (e.g., tri3b) -dec_script= # what decoding script to use (e.g., steps/decode_fmllr.sh) -phn_label=phnc -wrd_label=wrd -dec_suffix=word -dec_splits="train valid" -valid_split="valid" - -data_dir=$out_dir/data -wrd_data_dir=$out_dir/data_word - -lexicon_clean=$(mktemp) -cat $lexicon | sort | uniq > $lexicon_clean -local/prepare_lang_word.sh $w2v_dir/dict.${phn_label}.txt $data_dir $lexicon_clean && rm $lexicon_clean -local/prepare_lm.sh --langdir $data_dir/lang_word --lmdir $data_dir/lang_test_word $wrd_arpa_lm $data_dir - -for x in $dec_splits; do - x_gt=${x}_gt - mkdir -p $wrd_data_dir/$x_gt - cp $data_dir/$x_gt/{feats.scp,cmvn.scp,utt2spk,spk2utt} $wrd_data_dir/$x_gt/ - python local/copy_aligned_text.py < $w2v_dir/$x.$wrd_label > $wrd_data_dir/$x_gt/text -done - -local/decode.sh --nj 40 --graph_name graph${dec_suffix} --decode_suffix $dec_suffix \ - --val_sets "$dec_splits" --decode_script $dec_script \ - $out_dir/exp/$dec_exp $data_dir $data_dir/lang_test_word - -local/unsup_select_decode_word.sh \ - --split $valid_split --kenlm_path $wrd_arpa_lm_bin \ - --ref_txt $wrd_data_dir/${valid_split}_gt/text \ - --psd_txt $data_dir/${valid_split}/text \ - --dec_name decode${dec_suffix} --graph_name graph${dec_suffix} \ - --phonemize_lexicon $data_dir/local/dict_word/lexicon.txt \ - $out_dir/exp diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/decode_word_step2.sh b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/decode_word_step2.sh deleted file mode 100644 index 59a6cbb12539cf62658f8344f7be7cecf2e3380f..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/decode_word_step2.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -# prepare a new data directory of HMM word output - -. ./path.sh - -set -eu - -out_dir= # same as in train.sh -dec_lmparam= # LM hyperparameters (e.g., 7.0.0) - -dec_exp=tri3b # what HMM stage to decode (e.g., tri3b) -dec_suffix=word -dec_splits="train valid" -dec_data_dir=$out_dir/dec_data_word # where to write HMM output - -data_dir=$out_dir/data -wrd_data_dir=$out_dir/data_word - -for x in $dec_splits; do - mkdir -p $dec_data_dir/$x - cp $data_dir/$x/{feats.scp,cmvn.scp,utt2spk,spk2utt} $dec_data_dir/$x/ - - tra=$out_dir/exp/$dec_exp/decode${dec_suffix}_${x}/scoring/${dec_lmparam}.tra - cat $tra | utils/int2sym.pl -f 2- $data_dir/lang_word/words.txt | \ - sed 's:<UNK>::g' | sed 's:<SIL>::g' > $dec_data_dir/$x/text - utils/fix_data_dir.sh $dec_data_dir/$x - echo "WER on $x is" $(compute-wer ark:$wrd_data_dir/${x}_gt/text ark:$dec_data_dir/$x/text | cut -d" " -f2-) -done - diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/local/copy_aligned_text.py b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/local/copy_aligned_text.py deleted file mode 100644 index 5f4faa99218b0b30c980cad167c52b2297cd92c3..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/local/copy_aligned_text.py +++ /dev/null @@ -1,4 +0,0 @@ -import sys - -for idx, line in enumerate(sys.stdin): - print(f"utt{idx:010d} {line}", end='') \ No newline at end of file diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/local/decode.sh b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/local/decode.sh deleted file mode 100644 index 811cb63c88bb7cdd03b0a250ef2db32b5eaa50df..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/local/decode.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash - -set -u - -val_sets="dev_other" -graph_name=graph -decode_suffix="" -decode_script="steps/decode_fmllr.sh" -decode_args="" -nj=60 - -. ./cmd.sh -. ./path.sh -. parse_options.sh - -set -x -exp_dir=$1 -data_root=$2 -lang_test=$3 - -graph=$exp_dir/$graph_name - -if [ ! -d $graph ]; then - utils/mkgraph.sh $lang_test $exp_dir $graph -fi - -for part in $val_sets; do - dec_dir=$exp_dir/decode${decode_suffix}_${part} - if [ ! -d $dec_dir ]; then - echo "decoding $part for $exp_dir" - $decode_script --nj $nj --cmd "$decode_cmd" $decode_args \ - $graph $data_root/$part $dec_dir & - else - echo "$dec_dir exists. skip" - fi -done - -wait diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/local/prepare_data_from_w2v.py b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/local/prepare_data_from_w2v.py deleted file mode 100644 index 20d6c5caf6187f803a37f0f5b9967df363edd88d..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/local/prepare_data_from_w2v.py +++ /dev/null @@ -1,57 +0,0 @@ -import os - -import kaldi_io -import numpy as np - - -def get_parser(): - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("w2v_dir", help="wav2vec feature and text directory") - parser.add_argument("tar_root", help="output data directory in kaldi's format") - parser.add_argument("split", help="name of the subset") - parser.add_argument("--label", default="", help="if specified, copy labels too") - return parser - -def main(): - parser = get_parser() - args = parser.parse_args() - - tar_dir = os.path.join(args.tar_root, args.split) - os.makedirs(tar_dir, exist_ok=True) - - lengths_path = os.path.join(args.w2v_dir, f"{args.split}.lengths") - with open(lengths_path) as f: - lengths = [int(line.rstrip()) for line in f] - offsets = [0] + np.cumsum(lengths[:-1]).tolist() - feats = np.load( - os.path.join(args.w2v_dir, f"{args.split}.npy"), - mmap_mode="r" - ) - assert feats.shape[0] == sum(lengths), \ - f"lengths mismatch {feats.shape[0]} != {sum(lengths)}" - - ark_path = os.path.join(tar_dir, "feats.ark") - scp_path = os.path.join(tar_dir, "feats.scp") - wspec = f"ark:| copy-feats --compress=true ark:- ark,scp:{ark_path},{scp_path}" - with kaldi_io.open_or_fd(wspec, "wb") as f: - for idx, (offset, length) in enumerate(zip(offsets, lengths)): - feat = feats[offset:offset+length] - kaldi_io.write_mat(f, feat, key=f"utt{idx:010d}") - - u2s_path = os.path.join(tar_dir, "utt2spk") - s2u_path = os.path.join(tar_dir, "spk2utt") - with open(u2s_path, "w") as f_u2s, open(s2u_path, "w") as f_s2u: - for idx in range(len(lengths)): - f_u2s.write(f"utt{idx:010d} utt{idx:010d}\n") - f_s2u.write(f"utt{idx:010d} utt{idx:010d}\n") - - if bool(args.label): - lab_path = os.path.join(args.w2v_dir, f"{args.split}.{args.label}") - txt_path = os.path.join(tar_dir, "text") - with open(lab_path) as f_lab, open(txt_path, "w") as f_txt: - for idx, line in enumerate(f_lab): - f_txt.write(f"utt{idx:010d} {line}") - -if __name__ == "__main__": - main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/local/prepare_lang.sh b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/local/prepare_lang.sh deleted file mode 100644 index e9a80001eb47d5af863d6aab11a59362a59cef61..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/local/prepare_lang.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -sil_prob=0.5 -num_sil_states=3 -num_nonsil_states=1 - -. ./cmd.sh -. ./path.sh -. parse_options.sh - -set -eux - -dict=$1 -data_dir=$2 - -dict_dir=$data_dir/local/dict -tmplm_dir=$data_dir/local/lang_tmp -lm_dir=$data_dir/lang - -mkdir -p $dict_dir $tmplm_dir $lm_dir - -# prepare dict -echo "SIL" > $dict_dir/silence_phones.txt -echo "SIL" > $dict_dir/optional_silence.txt -awk '{print $1}' $dict > $dict_dir/nonsilence_phones.txt - -echo "SIL SIL" > $dict_dir/lexicon.txt -echo "<UNK> SIL" >> $dict_dir/lexicon.txt -awk '{print $1" "$1}' $dict >> $dict_dir/lexicon.txt - -echo "SIL" > $dict_dir/extra_questions.txt -awk '{printf $1" "} END {printf "\n"}' $dict >> $dict_dir/extra_questions.txt - -# prepare lang -utils/prepare_lang.sh --sil-prob $sil_prob --position-dependent-phones false \ - --num_sil_states $num_sil_states --num_nonsil_states $num_nonsil_states \ - $dict_dir "<UNK>" $tmplm_dir $lm_dir diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/local/prepare_lang_word.sh b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/local/prepare_lang_word.sh deleted file mode 100644 index a7ea3877beefe1d4d53f9f7e32b004d8ce01e22a..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/local/prepare_lang_word.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -num_sil_states=3 -num_nonsil_states=1 - -. ./cmd.sh -. ./path.sh -. parse_options.sh - -set -eux - -dict=$1 -data_dir=$2 -lexicon=$3 - -dict_dir=$data_dir/local/dict_word -tmplm_dir=$data_dir/local/lang_tmp_word -lm_dir=$data_dir/lang_word - -mkdir -p $dict_dir $tmplm_dir $lm_dir - -# prepare dict -echo "SIL" > $dict_dir/silence_phones.txt -echo "SIL" > $dict_dir/optional_silence.txt -awk '{print $1}' $dict > $dict_dir/nonsilence_phones.txt - -(echo "!SIL SIL"; echo "<UNK> SIL";) | cat - $lexicon > $dict_dir/lexicon.txt - -echo "SIL" > $dict_dir/extra_questions.txt -awk '{printf $1" "} END {printf "\n"}' $dict >> $dict_dir/extra_questions.txt - -# prepare lang -utils/prepare_lang.sh --position-dependent-phones false \ - --num_sil_states $num_sil_states --num_nonsil_states $num_nonsil_states \ - $dict_dir "<UNK>" $tmplm_dir $lm_dir diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/local/prepare_lm.sh b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/local/prepare_lm.sh deleted file mode 100644 index c2edcefede2da3b6a991b9c8fbc78c96d46d27cb..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/local/prepare_lm.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash - -langdir="" -lmdir="" - -. ./cmd.sh -. ./path.sh -. parse_options.sh - -arpa_lm=$1 -data=$2 - -if [ -z $langdir ]; then - langdir=$data/lang -fi -if [ -z $lmdir ]; then - lmdir=$data/lang_test -fi - -if [ ! -d $langdir ]; then - echo "$langdir not found. run local/prepare_lang.sh first" && exit 1 -fi - -mkdir -p $lmdir -cp -r $langdir/* $lmdir - -if [[ "$arpa_lm" == *.gz ]]; then - gunzip -c $arpa_lm | arpa2fst --disambig-symbol=#0 --read-symbol-table=$lmdir/words.txt - $lmdir/G.fst -else - arpa2fst --disambig-symbol=#0 --read-symbol-table=$lmdir/words.txt $arpa_lm $lmdir/G.fst -fi -fstisstochastic $lmdir/G.fst -utils/validate_lang.pl $lmdir || exit 1 - -echo "done preparing lm ($lmdir)" diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/local/score.sh b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/local/score.sh deleted file mode 100644 index cb5bbb7277bfb9f2d5440da0514bf7b16da8140d..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/local/score.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2012 Johns Hopkins University (Author: Daniel Povey) -# 2014 Guoguo Chen -# Apache 2.0 - -[ -f ./path.sh ] && . ./path.sh - -# begin configuration section. -cmd=run.pl -stage=0 -decode_mbr=true -word_ins_penalty=0.0,0.5,1.0 -min_lmwt=7 -max_lmwt=17 -iter=final -#end configuration section. - -[ -f ./path.sh ] && . ./path.sh -. parse_options.sh || exit 1; - -if [ $# -ne 3 ]; then - echo "Usage: local/score.sh [--cmd (run.pl|queue.pl...)] <data-dir> <lang-dir|graph-dir> <decode-dir>" - echo " Options:" - echo " --cmd (run.pl|queue.pl...) # specify how to run the sub-processes." - echo " --stage (0|1|2) # start scoring script from part-way through." - echo " --decode_mbr (true/false) # maximum bayes risk decoding (confusion network)." - echo " --min_lmwt <int> # minumum LM-weight for lattice rescoring " - echo " --max_lmwt <int> # maximum LM-weight for lattice rescoring " - exit 1; -fi - -data=$1 -lang_or_graph=$2 -dir=$3 - -symtab=$lang_or_graph/words.txt - -for f in $symtab $dir/lat.1.gz $data/text; do - [ ! -f $f ] && echo "score.sh: no such file $f" && exit 1; -done - -mkdir -p $dir/scoring/log - -cat $data/text | sed 's:<NOISE>::g' | sed 's:<SPOKEN_NOISE>::g' > $dir/scoring/test_filt.txt - -for wip in $(echo $word_ins_penalty | sed 's/,/ /g'); do - $cmd LMWT=$min_lmwt:$max_lmwt $dir/scoring/log/best_path.LMWT.$wip.log \ - lattice-scale --inv-acoustic-scale=LMWT "ark:gunzip -c $dir/lat.*.gz|" ark:- \| \ - lattice-add-penalty --word-ins-penalty=$wip ark:- ark:- \| \ - lattice-best-path --word-symbol-table=$symtab \ - ark:- ark,t:$dir/scoring/LMWT.$wip.tra || exit 1; -done - -# Note: the double level of quoting for the sed command -for wip in $(echo $word_ins_penalty | sed 's/,/ /g'); do - $cmd LMWT=$min_lmwt:$max_lmwt $dir/scoring/log/score.LMWT.$wip.log \ - cat $dir/scoring/LMWT.$wip.tra \| \ - utils/int2sym.pl -f 2- $symtab \| sed 's:\<UNK\>::g' \| \ - compute-wer --text --mode=present \ - ark:$dir/scoring/test_filt.txt ark,p:- ">&" $dir/wer_LMWT_$wip || exit 1; -done - -exit 0; diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/local/show_wer.sh b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/local/show_wer.sh deleted file mode 100644 index 9ecf1690c67f8a019009ef32d973fbd45b56c7ca..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/local/show_wer.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash - -split="dev_other" -ref_data="" -get_best_wer=true -dec_name="decode" -graph_name="graph" - -. ./cmd.sh -. ./path.sh -. parse_options.sh - -exp_root=$1 - -set -eu - -echo "==== WER w.r.t. pseudo transcript" -for x in $exp_root/*/${dec_name}_${split}*; do grep WER $x/wer_* 2>/dev/null | utils/best_wer.sh; done - - -if [ ! -z $ref_data ]; then - echo "==== WER w.r.t. real transcript (select based on pseudo WER)" - ref_txt=$ref_data/$split/text - for x in $exp_root/*/${dec_name}_${split}*; do - lang=$(dirname $x)/$graph_name - - lmwt=$( - grep WER $x/wer_* 2>/dev/null | utils/best_wer.sh | - sed 's/.*wer_\(.*\)$/\1/g' | sed 's/_/./g' - ) - tra=$x/scoring/$lmwt.tra - cat $tra | utils/int2sym.pl -f 2- $lang/words.txt | sed 's:<UNK>::g' | sed 's:<SIL>::g' | \ - compute-wer --text --mode=present \ - ark:$ref_txt ark,p:- 2> /dev/null | grep WER | xargs -I{} echo {} $tra - done -fi - -if [ ! -z $ref_data ] && $get_best_wer; then - echo "==== WER w.r.t. real transcript (select based on true WER)" - ref_txt=$ref_data/$split/text - for x in $exp_root/*/${dec_name}_${split}*; do - lang=$(dirname $x)/$graph_name - - for tra in $x/scoring/*.tra; do - cat $tra | utils/int2sym.pl -f 2- $lang/words.txt | sed 's:<UNK>::g' | sed 's:<SIL>::g' | \ - compute-wer --text --mode=present \ - ark:$ref_txt ark,p:- 2> /dev/null | grep WER | xargs -I{} echo {} $tra - done | sort -k2n | head -n1 - done -fi - -exit 0; diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/local/train_subset_lgbeam.sh b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/local/train_subset_lgbeam.sh deleted file mode 100644 index 913c1d8e4357c146026b86e78f0b16f921776441..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/local/train_subset_lgbeam.sh +++ /dev/null @@ -1,129 +0,0 @@ -#!/usr/bin/env bash - -out_root=/tmp -out_name=train_${RANDOM} -num_nonsil_states=1 - -valid="dev_other" -train="train" -mono_size="-1" # 2000 -tri1_size="-1" # 5000 -tri2b_size="-1" # 10000 -tri3b_size="-1" # 10000 - -# Acoustic model parameters -numLeavesTri1=2000 -numGaussTri1=10000 -numLeavesMLLT=2500 -numGaussMLLT=15000 -numLeavesSAT=2500 -numGaussSAT=15000 - -stage=1 -max_stage=1 - -. ./cmd.sh -. ./path.sh -. parse_options.sh - -data=$1 -lang=$2 -lang_test=$3 - -exp_root=$out_root/$out_name - -# you might not want to do this for interactive shells. -set -e - - -if [ $stage -le 1 ] && [ $max_stage -ge 1 ]; then - # train a monophone system - if [ ! $mono_size -eq -1 ]; then - utils/subset_data_dir.sh $data/$train $mono_size $data/${train}_${mono_size} - mono_train=${train}_${mono_size} - else - mono_train=${train} - fi - - steps/train_mono.sh --boost-silence 1.25 --nj 20 --cmd "$train_cmd" \ - --initial-beam 40 --regular-beam 60 --retry-beam 120 \ - $data/$mono_train $lang $exp_root/mono - - utils/mkgraph.sh $lang_test $exp_root/mono $exp_root/mono/graph - steps/decode.sh --nj 20 --cmd "$decode_cmd" \ - $exp_root/mono/graph $data/$valid $exp_root/mono/decode_$valid & -fi - - -if [ $stage -le 2 ] && [ $max_stage -ge 2 ]; then - # train a first delta + delta-delta triphone system on a subset of 5000 utterances - if [ ! $tri1_size -eq -1 ]; then - utils/subset_data_dir.sh $data/$train $tri1_size $data/${train}_${tri1_size} - tri1_train=${train}_${tri1_size} - else - tri1_train=${train} - fi - - steps/align_si.sh --boost-silence 1.25 --nj 10 --cmd "$train_cmd" \ - $data/$tri1_train $lang \ - $exp_root/mono $exp_root/mono_ali_${tri1_train} - - steps_gan/train_deltas.sh --boost-silence 1.25 --cmd "$train_cmd" \ - --num_nonsil_states $num_nonsil_states $numLeavesTri1 $numGaussTri1 \ - $data/$tri1_train $lang \ - $exp_root/mono_ali_${tri1_train} $exp_root/tri1 - - utils/mkgraph.sh $lang_test $exp_root/tri1 $exp_root/tri1/graph - steps/decode.sh --nj 20 --cmd "$decode_cmd" \ - $exp_root/tri1/graph $data/$valid $exp_root/tri1/decode_$valid & -fi - -if [ $stage -le 3 ] && [ $max_stage -ge 3 ]; then - # train an LDA+MLLT system. - if [ ! $tri2b_size -eq -1 ]; then - utils/subset_data_dir.sh $data/$train $tri2b_size $data/${train}_${tri2b_size} - tri2b_train=${train}_${tri2b_size} - else - tri2b_train=${train} - fi - - steps/align_si.sh --nj 10 --cmd "$train_cmd" \ - $data/$tri2b_train $lang \ - $exp_root/tri1 $exp_root/tri1_ali_${tri2b_train} - - steps_gan/train_lda_mllt.sh --cmd "$train_cmd" \ - --num_nonsil_states $num_nonsil_states \ - --splice-opts "--left-context=3 --right-context=3" $numLeavesMLLT $numGaussMLLT \ - $data/$tri2b_train $lang \ - $exp_root/tri1_ali_${tri2b_train} $exp_root/tri2b - - utils/mkgraph.sh $lang_test $exp_root/tri2b $exp_root/tri2b/graph - steps/decode.sh --nj 20 --cmd "$decode_cmd" \ - $exp_root/tri2b/graph $data/$valid $exp_root/tri2b/decode_$valid & -fi - - -if [ $stage -le 4 ] && [ $max_stage -ge 4 ]; then - # Train tri3b, which is LDA+MLLT+SAT on 10k utts - if [ ! $tri3b_size -eq -1 ]; then - utils/subset_data_dir.sh $data/$train $tri3b_size $data/${train}_${tri3b_size} - tri3b_train=${train}_${tri3b_size} - else - tri3b_train=${train} - fi - - steps/align_si.sh --nj 10 --cmd "$train_cmd" --use-graphs true \ - $data/$tri3b_train $lang \ - $exp_root/tri2b $exp_root/tri2b_ali_${tri2b_train} - - steps_gan/train_sat.sh --cmd "$train_cmd" \ - --num_nonsil_states $num_nonsil_states $numLeavesSAT $numGaussSAT \ - $data/$tri3b_train $lang \ - $exp_root/tri2b_ali_${tri2b_train} $exp_root/tri3b - - utils/mkgraph.sh $lang_test $exp_root/tri3b $exp_root/tri3b/graph - steps/decode_fmllr.sh --nj 20 --cmd "$decode_cmd" \ - $exp_root/tri3b/graph $data/$valid $exp_root/tri3b/decode_$valid & -fi - -wait diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/local/unsup_select.py b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/local/unsup_select.py deleted file mode 100644 index 742c38841edf988ed39abeb5b7d7ee9a4e505e62..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/local/unsup_select.py +++ /dev/null @@ -1,135 +0,0 @@ -""" -Implement unsupervised metric for decoding hyperparameter selection: - $$ alpha * LM_PPL + ViterbitUER(%) * 100 $$ -""" -import argparse -import logging -import math -import sys - -import editdistance -import kenlm -from g2p_en import G2p - -logging.root.setLevel(logging.INFO) -logging.basicConfig(stream=sys.stdout, level=logging.INFO) -logger = logging.getLogger(__name__) - - -def get_parser(): - parser = argparse.ArgumentParser() - parser.add_argument("ref_tra", help="reference pseudo labels") - parser.add_argument("hyp_tra", help="decoded pseudo labels to be assess") - parser.add_argument("--kenlm_path", default="/checkpoint/abaevski/data/speech/libri/librispeech_lm_novox.phnc_o5.bin", help="") - parser.add_argument("--uppercase", action="store_true", help="") - parser.add_argument("--skipwords", default="", help="") - parser.add_argument("--gt_tra", default="", help="ground truth pseudo labels for computing oracle WER") - parser.add_argument("--min_vt_uer", default=0.0, type=float) - parser.add_argument("--phonemize", action="store_true", help="phonemize word hypotheses, used when reference is phone transcript") - parser.add_argument("--phonemize_lexicon", default="", type=str, help="use a lexicon for phonemizing") - return parser - -def load_tra(tra_path): - with open(tra_path, "r") as f: - uid_to_tra = {} - for line in f: - toks = line.rstrip().split() - uid, tra = toks[0], " ".join(toks[1:]) - uid_to_tra[uid] = tra - logger.debug(f"loaded {len(uid_to_tra)} utterances from {tra_path}") - return uid_to_tra - -def load_lex(lex_path): - with open(lex_path, "r") as f: - w2p = {} - for line in f: - w, p = line.rstrip().split(None, 1) - w2p[w] = p.split() - return w2p - -def compute_wer(ref_uid_to_tra, hyp_uid_to_tra, g2p, g2p_dict): - d_cnt = 0 - w_cnt = 0 - w_cnt_h = 0 - for uid in hyp_uid_to_tra: - ref = ref_uid_to_tra[uid].split() - if g2p_dict is not None: - hyp = [] - for word in hyp_uid_to_tra[uid].split(): - if word in g2p_dict: - hyp = hyp + g2p_dict[word] - else: - logger.warning(f"{word} not in g2p_dict") - elif g2p is not None: - hyp = g2p(hyp_uid_to_tra[uid]) - hyp = [p for p in hyp if p != "'" and p != " "] - hyp = [p[:-1] if p[-1].isnumeric() else p for p in hyp] - else: - hyp = hyp_uid_to_tra[uid].split() - logger.debug(( - f"======================\n" - f"HYP: {' '.join(hyp)}\n" - f"REF: {' '.join(ref)}" - )) - d_cnt += editdistance.eval(ref, hyp) - w_cnt += len(ref) - w_cnt_h += len(hyp) - wer = float(d_cnt) / w_cnt - logger.debug(( - f"wer = {wer*100:.2f}%; num. of ref words = {w_cnt}; " - f"num. of hyp words = {w_cnt_h}; num. of sentences = {len(ref_uid_to_tra)}" - )) - return wer - -def compute_lm_ppl(hyp_uid_to_tra, score_fn): - lm_score = 0. - w_cnt = 0 - for hyp in hyp_uid_to_tra.values(): - cur_score = score_fn(hyp) - cur_cnt = len(hyp.split()) + 1 # plus one for </s> - lm_score += cur_score - w_cnt += cur_cnt - logger.debug(( - f"======================\n" - f"score sum/avg = {cur_score:.2f}/{cur_score/cur_cnt:.2f}\n" - f"hyp = {hyp}" - )) - lm_ppl = math.pow(10, -lm_score / w_cnt) - logger.debug(f"lm ppl = {lm_ppl:.2f}; num. of words = {w_cnt}") - return lm_ppl - -def main(): - args = get_parser().parse_args() - logger.debug(f"Args: {args}") - - ref_uid_to_tra = load_tra(args.ref_tra) - hyp_uid_to_tra = load_tra(args.hyp_tra) - assert not bool(set(hyp_uid_to_tra.keys()) - set(ref_uid_to_tra.keys())) - - lm = kenlm.Model(args.kenlm_path) - skipwords = set(args.skipwords.split(",")) - def compute_lm_score(s): - s = " ".join(w for w in s.split() if w not in skipwords) - s = s.upper() if args.uppercase else s - return lm.score(s) - - g2p, g2p_dict = None, None - if args.phonemize: - if args.phonemize_lexicon: - g2p_dict = load_lex(args.phonemize_lexicon) - else: - g2p = G2p() - - wer = compute_wer(ref_uid_to_tra, hyp_uid_to_tra, g2p, g2p_dict) - lm_ppl = compute_lm_ppl(hyp_uid_to_tra, compute_lm_score) - - gt_wer = -math.inf - if args.gt_tra: - gt_uid_to_tra = load_tra(args.gt_tra) - gt_wer = compute_wer(gt_uid_to_tra, hyp_uid_to_tra, None, None) - - score = math.log(lm_ppl) * max(wer, args.min_vt_uer) - logging.info(f"{args.hyp_tra}: score={score:.4f}; wer={wer*100:.2f}%; lm_ppl={lm_ppl:.4f}; gt_wer={gt_wer*100:.2f}%") - -if __name__ == "__main__": - main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/local/unsup_select_decode.sh b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/local/unsup_select_decode.sh deleted file mode 100644 index b34c5b6e0688914a53515162f817a93617b609e5..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/local/unsup_select_decode.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -split="dev_other" -ref_txt="" # ground truth transcript path -psd_txt="" # pseudo transcript path -get_best_wer=true -dec_name="decode" -graph_name="graph" -kenlm_path=/checkpoint/abaevski/data/speech/libri/librispeech_lm_novox.phnc_o6.bin - -. ./cmd.sh -. ./path.sh -. parse_options.sh - -exp_root=$1 -unsup_args="" -if [ $# -ge 2 ]; then - unsup_args=$2 -fi - -set -eu - -if [ ! -z $ref_txt ] && $get_best_wer; then - echo "==== WER w.r.t. real transcript (select based on unsupervised metric)" - for x in $exp_root/*/${dec_name}_${split}*; do - lang=$(dirname $x)/$graph_name - - ( - for tra in $x/scoring/*.tra; do - cat $tra | utils/int2sym.pl -f 2- $lang/words.txt | sed 's:<UNK>::g' | sed 's:<SIL>::g' > $tra.txt - python local/unsup_select.py $psd_txt $tra.txt --kenlm_path $kenlm_path --gt_tra $ref_txt $unsup_args - done 2>/dev/null | grep "score=" | sed 's/=/ /g' | sed 's/;//g' | sort -k3n | head -n1 - ) & - done -fi -wait - diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/local/unsup_select_decode_word.sh b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/local/unsup_select_decode_word.sh deleted file mode 100644 index c10a6b8809b77bca2b2c02df8b8702725bdd51c7..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/local/unsup_select_decode_word.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -split="dev_other" -ref_txt="" # ground truth transcript path -psd_txt="" # pseudo transcript path -get_best_wer=true -dec_name="decode" -graph_name="graph" -kenlm_path=/checkpoint/abaevski/data/speech/libri/librispeech_lm_novox.phnc_o6.bin -phonemize_lexicon="" - -. ./cmd.sh -. ./path.sh -. parse_options.sh -. /private/home/wnhsu/unsup_asr/fairseq-py-unsup/env.sh - -exp_root=$1 - -set -eu - -if [ ! -z $ref_txt ] && $get_best_wer; then - echo "==== WER w.r.t. real transcript (select based on unsupervised metric)" - for x in $exp_root/*/${dec_name}_${split}*; do - lang=$(dirname $x)/$graph_name - - for tra in $x/scoring/*.tra; do - cat $tra | utils/int2sym.pl -f 2- $lang/words.txt | sed 's:\<UNK\>::g' > $tra.txt - python local/unsup_select.py $psd_txt $tra.txt \ - --kenlm_path $kenlm_path --gt_tra $ref_txt --phonemize \ - --phonemize_lexicon "$phonemize_lexicon" - done | grep "score=" | sed 's/=/ /g' | sed 's/;//g' | sort -k3n | head -n1 - done -fi - - diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/path.sh b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/path.sh deleted file mode 100644 index 1a6fb5f891b55d9fd978cfe54565f112f7eedce7..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/path.sh +++ /dev/null @@ -1,5 +0,0 @@ -export KALDI_ROOT=`pwd`/../../.. -export PATH=$PWD/utils/:$KALDI_ROOT/tools/openfst/bin:$PWD:$PATH -[ ! -f $KALDI_ROOT/tools/config/common_path.sh ] && echo >&2 "The standard file $KALDI_ROOT/tools/config/common_path.sh is not present -> Exit!" && exit 1 -. $KALDI_ROOT/tools/config/common_path.sh -export LC_ALL=C diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/steps_gan/train_deltas.sh b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/steps_gan/train_deltas.sh deleted file mode 100644 index af68715ab0d87ae40666596d9d877d593684f8e2..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/steps_gan/train_deltas.sh +++ /dev/null @@ -1,175 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2012 Johns Hopkins University (Author: Daniel Povey) -# Apache 2.0 - -# Begin configuration. -stage=-4 # This allows restarting after partway, when something when wrong. -config= -cmd=run.pl -scale_opts="--transition-scale=1.0 --acoustic-scale=0.1 --self-loop-scale=0.1" -realign_iters="10 20 30"; -num_iters=35 # Number of iterations of training -max_iter_inc=25 # Last iter to increase #Gauss on. -beam=10 -careful=false -retry_beam=40 -boost_silence=1.0 # Factor by which to boost silence likelihoods in alignment -power=0.25 # Exponent for number of gaussians according to occurrence counts -cluster_thresh=-1 # for build-tree control final bottom-up clustering of leaves -norm_vars=false # deprecated. Prefer --cmvn-opts "--norm-vars=true" - # use the option --cmvn-opts "--norm-means=false" -cmvn_opts= -delta_opts= -context_opts= # use"--context-width=5 --central-position=2" for quinphone -num_nonsil_states=3 -# End configuration. - -echo "$0 $@" # Print the command line for logging - -[ -f path.sh ] && . ./path.sh; -. parse_options.sh || exit 1; - -if [ $# != 6 ]; then - echo "Usage: steps/train_deltas.sh <num-leaves> <tot-gauss> <data-dir> <lang-dir> <alignment-dir> <exp-dir>" - echo "e.g.: steps/train_deltas.sh 2000 10000 data/train_si84_half data/lang exp/mono_ali exp/tri1" - echo "main options (for others, see top of script file)" - echo " --cmd (utils/run.pl|utils/queue.pl <queue opts>) # how to run jobs." - echo " --config <config-file> # config containing options" - echo " --stage <stage> # stage to do partial re-run from." - exit 1; -fi - -numleaves=$1 -totgauss=$2 -data=$3 -lang=$4 -alidir=$5 -dir=$6 - -for f in $alidir/final.mdl $alidir/ali.1.gz $data/feats.scp $lang/phones.txt; do - [ ! -f $f ] && echo "train_deltas.sh: no such file $f" && exit 1; -done - -numgauss=$numleaves -incgauss=$[($totgauss-$numgauss)/$max_iter_inc] # per-iter increment for #Gauss -oov=`cat $lang/oov.int` || exit 1; -ciphonelist=`cat $lang/phones/context_indep.csl` || exit 1; -nj=`cat $alidir/num_jobs` || exit 1; -mkdir -p $dir/log -echo $nj > $dir/num_jobs - -utils/lang/check_phones_compatible.sh $lang/phones.txt $alidir/phones.txt || exit 1; -cp $lang/phones.txt $dir || exit 1; - -sdata=$data/split$nj; -split_data.sh $data $nj || exit 1; - - -[ $(cat $alidir/cmvn_opts 2>/dev/null | wc -c) -gt 1 ] && [ -z "$cmvn_opts" ] && \ - echo "$0: warning: ignoring CMVN options from source directory $alidir" -$norm_vars && cmvn_opts="--norm-vars=true $cmvn_opts" -echo $cmvn_opts > $dir/cmvn_opts # keep track of options to CMVN. -[ ! -z $delta_opts ] && echo $delta_opts > $dir/delta_opts - -feats="ark,s,cs:apply-cmvn $cmvn_opts --utt2spk=ark:$sdata/JOB/utt2spk scp:$sdata/JOB/cmvn.scp scp:$sdata/JOB/feats.scp ark:- | add-deltas $delta_opts ark:- ark:- |" - -rm $dir/.error 2>/dev/null - -if [ $stage -le -3 ]; then - echo "$0: accumulating tree stats" - $cmd JOB=1:$nj $dir/log/acc_tree.JOB.log \ - acc-tree-stats $context_opts \ - --ci-phones=$ciphonelist $alidir/final.mdl "$feats" \ - "ark:gunzip -c $alidir/ali.JOB.gz|" $dir/JOB.treeacc || exit 1; - sum-tree-stats $dir/treeacc $dir/*.treeacc 2>$dir/log/sum_tree_acc.log || exit 1; - rm $dir/*.treeacc -fi - -if [ $stage -le -2 ]; then - echo "$0: getting questions for tree-building, via clustering" - # preparing questions, roots file... - cluster-phones --pdf-class-list=$(($num_nonsil_states / 2)) $context_opts \ - $dir/treeacc $lang/phones/sets.int \ - $dir/questions.int 2> $dir/log/questions.log || exit 1; - cat $lang/phones/extra_questions.int >> $dir/questions.int - compile-questions $context_opts $lang/topo $dir/questions.int \ - $dir/questions.qst 2>$dir/log/compile_questions.log || exit 1; - - echo "$0: building the tree" - $cmd $dir/log/build_tree.log \ - build-tree $context_opts --verbose=1 --max-leaves=$numleaves \ - --cluster-thresh=$cluster_thresh $dir/treeacc $lang/phones/roots.int \ - $dir/questions.qst $lang/topo $dir/tree || exit 1; - - $cmd $dir/log/init_model.log \ - gmm-init-model --write-occs=$dir/1.occs \ - $dir/tree $dir/treeacc $lang/topo $dir/1.mdl || exit 1; - if grep 'no stats' $dir/log/init_model.log; then - echo "** The warnings above about 'no stats' generally mean you have phones **" - echo "** (or groups of phones) in your phone set that had no corresponding data. **" - echo "** You should probably figure out whether something went wrong, **" - echo "** or whether your data just doesn't happen to have examples of those **" - echo "** phones. **" - fi - - gmm-mixup --mix-up=$numgauss $dir/1.mdl $dir/1.occs $dir/1.mdl 2>$dir/log/mixup.log || exit 1; - rm $dir/treeacc -fi - -if [ $stage -le -1 ]; then - # Convert the alignments. - echo "$0: converting alignments from $alidir to use current tree" - $cmd JOB=1:$nj $dir/log/convert.JOB.log \ - convert-ali $alidir/final.mdl $dir/1.mdl $dir/tree \ - "ark:gunzip -c $alidir/ali.JOB.gz|" "ark:|gzip -c >$dir/ali.JOB.gz" || exit 1; -fi - -if [ $stage -le 0 ]; then - echo "$0: compiling graphs of transcripts" - $cmd JOB=1:$nj $dir/log/compile_graphs.JOB.log \ - compile-train-graphs --read-disambig-syms=$lang/phones/disambig.int $dir/tree $dir/1.mdl $lang/L.fst \ - "ark:utils/sym2int.pl --map-oov $oov -f 2- $lang/words.txt < $sdata/JOB/text |" \ - "ark:|gzip -c >$dir/fsts.JOB.gz" || exit 1; -fi - -x=1 -while [ $x -lt $num_iters ]; do - echo "$0: training pass $x" - if [ $stage -le $x ]; then - if echo $realign_iters | grep -w $x >/dev/null; then - echo "$0: aligning data" - mdl="gmm-boost-silence --boost=$boost_silence `cat $lang/phones/optional_silence.csl` $dir/$x.mdl - |" - $cmd JOB=1:$nj $dir/log/align.$x.JOB.log \ - gmm-align-compiled $scale_opts --beam=$beam --retry-beam=$retry_beam --careful=$careful "$mdl" \ - "ark:gunzip -c $dir/fsts.JOB.gz|" "$feats" \ - "ark:|gzip -c >$dir/ali.JOB.gz" || exit 1; - fi - $cmd JOB=1:$nj $dir/log/acc.$x.JOB.log \ - gmm-acc-stats-ali $dir/$x.mdl "$feats" \ - "ark,s,cs:gunzip -c $dir/ali.JOB.gz|" $dir/$x.JOB.acc || exit 1; - $cmd $dir/log/update.$x.log \ - gmm-est --mix-up=$numgauss --power=$power \ - --write-occs=$dir/$[$x+1].occs $dir/$x.mdl \ - "gmm-sum-accs - $dir/$x.*.acc |" $dir/$[$x+1].mdl || exit 1; - rm $dir/$x.mdl $dir/$x.*.acc - rm $dir/$x.occs - fi - [ $x -le $max_iter_inc ] && numgauss=$[$numgauss+$incgauss]; - x=$[$x+1]; -done - -rm $dir/final.mdl $dir/final.occs 2>/dev/null -ln -s $x.mdl $dir/final.mdl -ln -s $x.occs $dir/final.occs - -steps/diagnostic/analyze_alignments.sh --cmd "$cmd" $lang $dir - -# Summarize warning messages... -utils/summarize_warnings.pl $dir/log - -steps/info/gmm_dir_info.pl $dir - -echo "$0: Done training system with delta+delta-delta features in $dir" - -exit 0 diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/steps_gan/train_lda_mllt.sh b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/steps_gan/train_lda_mllt.sh deleted file mode 100644 index 9d8c319ce848e431ec47a3548156347ae3b50ced..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/steps_gan/train_lda_mllt.sh +++ /dev/null @@ -1,239 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2012 Johns Hopkins University (Author: Daniel Povey) -# -# LDA+MLLT refers to the way we transform the features after computing -# the MFCCs: we splice across several frames, reduce the dimension (to 40 -# by default) using Linear Discriminant Analysis), and then later estimate, -# over multiple iterations, a diagonalizing transform known as MLLT or STC. -# See http://kaldi-asr.org/doc/transform.html for more explanation. -# -# Apache 2.0. - -# Begin configuration. -cmd=run.pl -config= -stage=-5 -scale_opts="--transition-scale=1.0 --acoustic-scale=0.1 --self-loop-scale=0.1" -realign_iters="10 20 30"; -mllt_iters="2 4 6 12"; -num_iters=35 # Number of iterations of training -max_iter_inc=25 # Last iter to increase #Gauss on. -dim=40 -beam=10 -retry_beam=40 -careful=false -boost_silence=1.0 # Factor by which to boost silence likelihoods in alignment -power=0.25 # Exponent for number of gaussians according to occurrence counts -randprune=4.0 # This is approximately the ratio by which we will speed up the - # LDA and MLLT calculations via randomized pruning. -splice_opts= -cluster_thresh=-1 # for build-tree control final bottom-up clustering of leaves -norm_vars=false # deprecated. Prefer --cmvn-opts "--norm-vars=false" -cmvn_opts= -context_opts= # use "--context-width=5 --central-position=2" for quinphone. -# End configuration. -train_tree=true # if false, don't actually train the tree. -use_lda_mat= # If supplied, use this LDA[+MLLT] matrix. -num_nonsil_states=3 - -echo "$0 $@" # Print the command line for logging - -[ -f path.sh ] && . ./path.sh -. parse_options.sh || exit 1; - -if [ $# != 6 ]; then - echo "Usage: steps/train_lda_mllt.sh [options] <#leaves> <#gauss> <data> <lang> <alignments> <dir>" - echo " e.g.: steps/train_lda_mllt.sh 2500 15000 data/train_si84 data/lang exp/tri1_ali_si84 exp/tri2b" - echo "Main options (for others, see top of script file)" - echo " --cmd (utils/run.pl|utils/queue.pl <queue opts>) # how to run jobs." - echo " --config <config-file> # config containing options" - echo " --stage <stage> # stage to do partial re-run from." - exit 1; -fi - -numleaves=$1 -totgauss=$2 -data=$3 -lang=$4 -alidir=$5 -dir=$6 - -for f in $alidir/final.mdl $alidir/ali.1.gz $data/feats.scp $lang/phones.txt; do - [ ! -f $f ] && echo "train_lda_mllt.sh: no such file $f" && exit 1; -done - -numgauss=$numleaves -incgauss=$[($totgauss-$numgauss)/$max_iter_inc] # per-iter #gauss increment -oov=`cat $lang/oov.int` || exit 1; -nj=`cat $alidir/num_jobs` || exit 1; -silphonelist=`cat $lang/phones/silence.csl` || exit 1; -ciphonelist=`cat $lang/phones/context_indep.csl` || exit 1; - -mkdir -p $dir/log - -utils/lang/check_phones_compatible.sh $lang/phones.txt $alidir/phones.txt || exit 1; -cp $lang/phones.txt $dir || exit 1; - -echo $nj >$dir/num_jobs -echo "$splice_opts" >$dir/splice_opts # keep track of frame-splicing options - # so that later stages of system building can know what they were. - - -[ $(cat $alidir/cmvn_opts 2>/dev/null | wc -c) -gt 1 ] && [ -z "$cmvn_opts" ] && \ - echo "$0: warning: ignoring CMVN options from source directory $alidir" -$norm_vars && cmvn_opts="--norm-vars=true $cmvn_opts" -echo $cmvn_opts > $dir/cmvn_opts # keep track of options to CMVN. - -sdata=$data/split$nj; -split_data.sh $data $nj || exit 1; - -splicedfeats="ark,s,cs:apply-cmvn $cmvn_opts --utt2spk=ark:$sdata/JOB/utt2spk scp:$sdata/JOB/cmvn.scp scp:$sdata/JOB/feats.scp ark:- | splice-feats $splice_opts ark:- ark:- |" -# Note: $feats gets overwritten later in the script. -feats="$splicedfeats transform-feats $dir/0.mat ark:- ark:- |" - - - -if [ $stage -le -5 ]; then - if [ -z "$use_lda_mat" ]; then - echo "$0: Accumulating LDA statistics." - rm $dir/lda.*.acc 2>/dev/null - $cmd JOB=1:$nj $dir/log/lda_acc.JOB.log \ - ali-to-post "ark:gunzip -c $alidir/ali.JOB.gz|" ark:- \| \ - weight-silence-post 0.0 $silphonelist $alidir/final.mdl ark:- ark:- \| \ - acc-lda --rand-prune=$randprune $alidir/final.mdl "$splicedfeats" ark,s,cs:- \ - $dir/lda.JOB.acc || exit 1; - est-lda --write-full-matrix=$dir/full.mat --dim=$dim $dir/0.mat $dir/lda.*.acc \ - 2>$dir/log/lda_est.log || exit 1; - rm $dir/lda.*.acc - else - echo "$0: Using supplied LDA matrix $use_lda_mat" - cp $use_lda_mat $dir/0.mat || exit 1; - [ ! -z "$mllt_iters" ] && \ - echo "$0: Warning: using supplied LDA matrix $use_lda_mat but we will do MLLT," && \ - echo " which you might not want; to disable MLLT, specify --mllt-iters ''" && \ - sleep 5 - fi -fi - -cur_lda_iter=0 - -if [ $stage -le -4 ] && $train_tree; then - echo "$0: Accumulating tree stats" - $cmd JOB=1:$nj $dir/log/acc_tree.JOB.log \ - acc-tree-stats $context_opts \ - --ci-phones=$ciphonelist $alidir/final.mdl "$feats" \ - "ark:gunzip -c $alidir/ali.JOB.gz|" $dir/JOB.treeacc || exit 1; - [ `ls $dir/*.treeacc | wc -w` -ne "$nj" ] && echo "$0: Wrong #tree-accs" && exit 1; - $cmd $dir/log/sum_tree_acc.log \ - sum-tree-stats $dir/treeacc $dir/*.treeacc || exit 1; - rm $dir/*.treeacc -fi - - -if [ $stage -le -3 ] && $train_tree; then - echo "$0: Getting questions for tree clustering." - # preparing questions, roots file... - cluster-phones --pdf-class-list=$(($num_nonsil_states / 2)) $context_opts $dir/treeacc $lang/phones/sets.int \ - $dir/questions.int 2> $dir/log/questions.log || exit 1; - cat $lang/phones/extra_questions.int >> $dir/questions.int - compile-questions $context_opts $lang/topo $dir/questions.int \ - $dir/questions.qst 2>$dir/log/compile_questions.log || exit 1; - - echo "$0: Building the tree" - $cmd $dir/log/build_tree.log \ - build-tree $context_opts --verbose=1 --max-leaves=$numleaves \ - --cluster-thresh=$cluster_thresh $dir/treeacc $lang/phones/roots.int \ - $dir/questions.qst $lang/topo $dir/tree || exit 1; -fi - -if [ $stage -le -2 ]; then - echo "$0: Initializing the model" - if $train_tree; then - gmm-init-model --write-occs=$dir/1.occs \ - $dir/tree $dir/treeacc $lang/topo $dir/1.mdl 2> $dir/log/init_model.log || exit 1; - grep 'no stats' $dir/log/init_model.log && echo "This is a bad warning."; - rm $dir/treeacc - else - cp $alidir/tree $dir/ || exit 1; - $cmd JOB=1 $dir/log/init_model.log \ - gmm-init-model-flat $dir/tree $lang/topo $dir/1.mdl \ - "$feats subset-feats ark:- ark:-|" || exit 1; - fi -fi - - -if [ $stage -le -1 ]; then - # Convert the alignments. - echo "$0: Converting alignments from $alidir to use current tree" - $cmd JOB=1:$nj $dir/log/convert.JOB.log \ - convert-ali $alidir/final.mdl $dir/1.mdl $dir/tree \ - "ark:gunzip -c $alidir/ali.JOB.gz|" "ark:|gzip -c >$dir/ali.JOB.gz" || exit 1; -fi - -if [ $stage -le 0 ] && [ "$realign_iters" != "" ]; then - echo "$0: Compiling graphs of transcripts" - $cmd JOB=1:$nj $dir/log/compile_graphs.JOB.log \ - compile-train-graphs --read-disambig-syms=$lang/phones/disambig.int $dir/tree $dir/1.mdl $lang/L.fst \ - "ark:utils/sym2int.pl --map-oov $oov -f 2- $lang/words.txt < $data/split$nj/JOB/text |" \ - "ark:|gzip -c >$dir/fsts.JOB.gz" || exit 1; -fi - - -x=1 -while [ $x -lt $num_iters ]; do - echo Training pass $x - if echo $realign_iters | grep -w $x >/dev/null && [ $stage -le $x ]; then - echo Aligning data - mdl="gmm-boost-silence --boost=$boost_silence `cat $lang/phones/optional_silence.csl` $dir/$x.mdl - |" - $cmd JOB=1:$nj $dir/log/align.$x.JOB.log \ - gmm-align-compiled $scale_opts --beam=$beam --retry-beam=$retry_beam --careful=$careful "$mdl" \ - "ark:gunzip -c $dir/fsts.JOB.gz|" "$feats" \ - "ark:|gzip -c >$dir/ali.JOB.gz" || exit 1; - fi - if echo $mllt_iters | grep -w $x >/dev/null; then - if [ $stage -le $x ]; then - echo "$0: Estimating MLLT" - $cmd JOB=1:$nj $dir/log/macc.$x.JOB.log \ - ali-to-post "ark:gunzip -c $dir/ali.JOB.gz|" ark:- \| \ - weight-silence-post 0.0 $silphonelist $dir/$x.mdl ark:- ark:- \| \ - gmm-acc-mllt --rand-prune=$randprune $dir/$x.mdl "$feats" ark:- $dir/$x.JOB.macc \ - || exit 1; - est-mllt $dir/$x.mat.new $dir/$x.*.macc 2> $dir/log/mupdate.$x.log || exit 1; - gmm-transform-means $dir/$x.mat.new $dir/$x.mdl $dir/$x.mdl \ - 2> $dir/log/transform_means.$x.log || exit 1; - compose-transforms --print-args=false $dir/$x.mat.new $dir/$cur_lda_iter.mat $dir/$x.mat || exit 1; - rm $dir/$x.*.macc - fi - feats="$splicedfeats transform-feats $dir/$x.mat ark:- ark:- |" - cur_lda_iter=$x - fi - - if [ $stage -le $x ]; then - $cmd JOB=1:$nj $dir/log/acc.$x.JOB.log \ - gmm-acc-stats-ali $dir/$x.mdl "$feats" \ - "ark,s,cs:gunzip -c $dir/ali.JOB.gz|" $dir/$x.JOB.acc || exit 1; - $cmd $dir/log/update.$x.log \ - gmm-est --write-occs=$dir/$[$x+1].occs --mix-up=$numgauss --power=$power \ - $dir/$x.mdl "gmm-sum-accs - $dir/$x.*.acc |" $dir/$[$x+1].mdl || exit 1; - rm $dir/$x.mdl $dir/$x.*.acc $dir/$x.occs - fi - [ $x -le $max_iter_inc ] && numgauss=$[$numgauss+$incgauss]; - x=$[$x+1]; -done - -rm $dir/final.{mdl,mat,occs} 2>/dev/null -ln -s $x.mdl $dir/final.mdl -ln -s $x.occs $dir/final.occs -ln -s $cur_lda_iter.mat $dir/final.mat - -steps/diagnostic/analyze_alignments.sh --cmd "$cmd" $lang $dir - -# Summarize warning messages... -utils/summarize_warnings.pl $dir/log - -steps/info/gmm_dir_info.pl $dir - -echo "$0: Done training system with LDA+MLLT features in $dir" - -exit 0 diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/steps_gan/train_sat.sh b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/steps_gan/train_sat.sh deleted file mode 100644 index f75afafb1c4ad04ee71ab8541064ab0477430616..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/steps_gan/train_sat.sh +++ /dev/null @@ -1,281 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2012 Johns Hopkins University (Author: Daniel Povey). Apache 2.0. - - -# This does Speaker Adapted Training (SAT), i.e. train on -# fMLLR-adapted features. It can be done on top of either LDA+MLLT, or -# delta and delta-delta features. If there are no transforms supplied -# in the alignment directory, it will estimate transforms itself before -# building the tree (and in any case, it estimates transforms a number -# of times during training). - - -# Begin configuration section. -stage=-5 -exit_stage=-100 # you can use this to require it to exit at the - # beginning of a specific stage. Not all values are - # supported. -fmllr_update_type=full -cmd=run.pl -scale_opts="--transition-scale=1.0 --acoustic-scale=0.1 --self-loop-scale=0.1" -beam=10 -retry_beam=40 -careful=false -boost_silence=1.0 # Factor by which to boost silence likelihoods in alignment -context_opts= # e.g. set this to "--context-width 5 --central-position 2" for quinphone. -realign_iters="10 20 30"; -fmllr_iters="2 4 6 12"; -silence_weight=0.0 # Weight on silence in fMLLR estimation. -num_iters=35 # Number of iterations of training -max_iter_inc=25 # Last iter to increase #Gauss on. -power=0.2 # Exponent for number of gaussians according to occurrence counts -cluster_thresh=-1 # for build-tree control final bottom-up clustering of leaves -phone_map= -train_tree=true -tree_stats_opts= -cluster_phones_opts= -compile_questions_opts= -# End configuration section. -num_nonsil_states=3 - -echo "$0 $@" # Print the command line for logging - -[ -f path.sh ] && . ./path.sh -. parse_options.sh || exit 1; - -if [ $# != 6 ]; then - echo "Usage: steps/train_sat.sh <#leaves> <#gauss> <data> <lang> <ali-dir> <exp-dir>" - echo " e.g.: steps/train_sat.sh 2500 15000 data/train_si84 data/lang exp/tri2b_ali_si84 exp/tri3b" - echo "Main options (for others, see top of script file)" - echo " --cmd (utils/run.pl|utils/queue.pl <queue opts>) # how to run jobs." - echo " --config <config-file> # config containing options" - echo " --stage <stage> # stage to do partial re-run from." - exit 1; -fi - -numleaves=$1 -totgauss=$2 -data=$3 -lang=$4 -alidir=$5 -dir=$6 - -for f in $data/feats.scp $lang/phones.txt $alidir/final.mdl $alidir/ali.1.gz; do - [ ! -f $f ] && echo "train_sat.sh: no such file $f" && exit 1; -done - -numgauss=$numleaves -incgauss=$[($totgauss-$numgauss)/$max_iter_inc] # per-iter #gauss increment -oov=`cat $lang/oov.int` -nj=`cat $alidir/num_jobs` || exit 1; -silphonelist=`cat $lang/phones/silence.csl` -ciphonelist=`cat $lang/phones/context_indep.csl` || exit 1; -sdata=$data/split$nj; -splice_opts=`cat $alidir/splice_opts 2>/dev/null` # frame-splicing options. -cmvn_opts=`cat $alidir/cmvn_opts 2>/dev/null` -delta_opts=`cat $alidir/delta_opts 2>/dev/null` -phone_map_opt= -[ ! -z "$phone_map" ] && phone_map_opt="--phone-map='$phone_map'" - -mkdir -p $dir/log -cp $alidir/splice_opts $dir 2>/dev/null # frame-splicing options. -cp $alidir/cmvn_opts $dir 2>/dev/null # cmn/cmvn option. -cp $alidir/delta_opts $dir 2>/dev/null # delta option. - -utils/lang/check_phones_compatible.sh $lang/phones.txt $alidir/phones.txt || exit 1; -cp $lang/phones.txt $dir || exit 1; - -echo $nj >$dir/num_jobs -[[ -d $sdata && $data/feats.scp -ot $sdata ]] || split_data.sh $data $nj || exit 1; - -# Set up features. - -if [ -f $alidir/final.mat ]; then feat_type=lda; else feat_type=delta; fi -echo "$0: feature type is $feat_type" - -## Set up speaker-independent features. -case $feat_type in - delta) sifeats="ark,s,cs:apply-cmvn $cmvn_opts --utt2spk=ark:$sdata/JOB/utt2spk scp:$sdata/JOB/cmvn.scp scp:$sdata/JOB/feats.scp ark:- | add-deltas $delta_opts ark:- ark:- |";; - lda) sifeats="ark,s,cs:apply-cmvn $cmvn_opts --utt2spk=ark:$sdata/JOB/utt2spk scp:$sdata/JOB/cmvn.scp scp:$sdata/JOB/feats.scp ark:- | splice-feats $splice_opts ark:- ark:- | transform-feats $alidir/final.mat ark:- ark:- |" - cp $alidir/final.mat $dir - cp $alidir/full.mat $dir 2>/dev/null - ;; - *) echo "$0: invalid feature type $feat_type" && exit 1; -esac - -## Get initial fMLLR transforms (possibly from alignment dir) -if [ -f $alidir/trans.1 ]; then - echo "$0: Using transforms from $alidir" - feats="$sifeats transform-feats --utt2spk=ark:$sdata/JOB/utt2spk ark,s,cs:$alidir/trans.JOB ark:- ark:- |" - cur_trans_dir=$alidir -else - if [ $stage -le -5 ]; then - echo "$0: obtaining initial fMLLR transforms since not present in $alidir" - # The next line is necessary because of $silphonelist otherwise being incorrect; would require - # old $lang dir which would require another option. Not needed anyway. - [ ! -z "$phone_map" ] && \ - echo "$0: error: you must provide transforms if you use the --phone-map option." && exit 1; - $cmd JOB=1:$nj $dir/log/fmllr.0.JOB.log \ - ali-to-post "ark:gunzip -c $alidir/ali.JOB.gz|" ark:- \| \ - weight-silence-post $silence_weight $silphonelist $alidir/final.mdl ark:- ark:- \| \ - gmm-est-fmllr --fmllr-update-type=$fmllr_update_type \ - --spk2utt=ark:$sdata/JOB/spk2utt $alidir/final.mdl "$sifeats" \ - ark:- ark:$dir/trans.JOB || exit 1; - fi - feats="$sifeats transform-feats --utt2spk=ark:$sdata/JOB/utt2spk ark,s,cs:$dir/trans.JOB ark:- ark:- |" - cur_trans_dir=$dir -fi - -if [ $stage -le -4 ] && $train_tree; then - # Get tree stats. - echo "$0: Accumulating tree stats" - $cmd JOB=1:$nj $dir/log/acc_tree.JOB.log \ - acc-tree-stats $context_opts $tree_stats_opts $phone_map_opt --ci-phones=$ciphonelist $alidir/final.mdl "$feats" \ - "ark:gunzip -c $alidir/ali.JOB.gz|" $dir/JOB.treeacc || exit 1; - [ "`ls $dir/*.treeacc | wc -w`" -ne "$nj" ] && echo "$0: Wrong #tree-accs" && exit 1; - $cmd $dir/log/sum_tree_acc.log \ - sum-tree-stats $dir/treeacc $dir/*.treeacc || exit 1; - rm $dir/*.treeacc -fi - -if [ $stage -le -3 ] && $train_tree; then - echo "$0: Getting questions for tree clustering." - # preparing questions, roots file... - cluster-phones --pdf-class-list=$(($num_nonsil_states / 2)) \ - $cluster_phones_opts $context_opts \ - $dir/treeacc $lang/phones/sets.int $dir/questions.int 2>$dir/log/questions.log || exit 1; - cat $lang/phones/extra_questions.int >> $dir/questions.int - compile-questions $context_opts $compile_questions_opts $lang/topo $dir/questions.int $dir/questions.qst 2>$dir/log/compile_questions.log || exit 1; - - echo "$0: Building the tree" - $cmd $dir/log/build_tree.log \ - build-tree $context_opts --verbose=1 --max-leaves=$numleaves \ - --cluster-thresh=$cluster_thresh $dir/treeacc $lang/phones/roots.int \ - $dir/questions.qst $lang/topo $dir/tree || exit 1; -fi - -if [ $stage -le -2 ]; then - echo "$0: Initializing the model" - if $train_tree; then - gmm-init-model --write-occs=$dir/1.occs \ - $dir/tree $dir/treeacc $lang/topo $dir/1.mdl 2> $dir/log/init_model.log || exit 1; - grep 'no stats' $dir/log/init_model.log && echo "This is a bad warning."; - rm $dir/treeacc - else - cp $alidir/tree $dir/ || exit 1; - $cmd JOB=1 $dir/log/init_model.log \ - gmm-init-model-flat $dir/tree $lang/topo $dir/1.mdl \ - "$feats subset-feats ark:- ark:-|" || exit 1; - fi -fi - -if [ $stage -le -1 ]; then - # Convert the alignments. - echo "$0: Converting alignments from $alidir to use current tree" - $cmd JOB=1:$nj $dir/log/convert.JOB.log \ - convert-ali $phone_map_opt $alidir/final.mdl $dir/1.mdl $dir/tree \ - "ark:gunzip -c $alidir/ali.JOB.gz|" "ark:|gzip -c >$dir/ali.JOB.gz" || exit 1; -fi - -[ "$exit_stage" -eq 0 ] && echo "$0: Exiting early: --exit-stage $exit_stage" && exit 0; - -if [ $stage -le 0 ] && [ "$realign_iters" != "" ]; then - echo "$0: Compiling graphs of transcripts" - $cmd JOB=1:$nj $dir/log/compile_graphs.JOB.log \ - compile-train-graphs --read-disambig-syms=$lang/phones/disambig.int $dir/tree $dir/1.mdl $lang/L.fst \ - "ark:utils/sym2int.pl --map-oov $oov -f 2- $lang/words.txt < $sdata/JOB/text |" \ - "ark:|gzip -c >$dir/fsts.JOB.gz" || exit 1; -fi - -x=1 -while [ $x -lt $num_iters ]; do - echo Pass $x - if echo $realign_iters | grep -w $x >/dev/null && [ $stage -le $x ]; then - echo Aligning data - mdl="gmm-boost-silence --boost=$boost_silence `cat $lang/phones/optional_silence.csl` $dir/$x.mdl - |" - $cmd JOB=1:$nj $dir/log/align.$x.JOB.log \ - gmm-align-compiled $scale_opts --beam=$beam --retry-beam=$retry_beam --careful=$careful "$mdl" \ - "ark:gunzip -c $dir/fsts.JOB.gz|" "$feats" \ - "ark:|gzip -c >$dir/ali.JOB.gz" || exit 1; - fi - - if echo $fmllr_iters | grep -w $x >/dev/null; then - if [ $stage -le $x ]; then - echo Estimating fMLLR transforms - # We estimate a transform that's additional to the previous transform; - # we'll compose them. - $cmd JOB=1:$nj $dir/log/fmllr.$x.JOB.log \ - ali-to-post "ark:gunzip -c $dir/ali.JOB.gz|" ark:- \| \ - weight-silence-post $silence_weight $silphonelist $dir/$x.mdl ark:- ark:- \| \ - gmm-est-fmllr --fmllr-update-type=$fmllr_update_type \ - --spk2utt=ark:$sdata/JOB/spk2utt $dir/$x.mdl \ - "$feats" ark:- ark:$dir/tmp_trans.JOB || exit 1; - for n in `seq $nj`; do - ! ( compose-transforms --b-is-affine=true \ - ark:$dir/tmp_trans.$n ark:$cur_trans_dir/trans.$n ark:$dir/composed_trans.$n \ - && mv $dir/composed_trans.$n $dir/trans.$n && \ - rm $dir/tmp_trans.$n ) 2>$dir/log/compose_transforms.$x.log \ - && echo "$0: Error composing transforms" && exit 1; - done - fi - feats="$sifeats transform-feats --utt2spk=ark:$sdata/JOB/utt2spk ark:$dir/trans.JOB ark:- ark:- |" - cur_trans_dir=$dir - fi - - if [ $stage -le $x ]; then - $cmd JOB=1:$nj $dir/log/acc.$x.JOB.log \ - gmm-acc-stats-ali $dir/$x.mdl "$feats" \ - "ark,s,cs:gunzip -c $dir/ali.JOB.gz|" $dir/$x.JOB.acc || exit 1; - [ `ls $dir/$x.*.acc | wc -w` -ne "$nj" ] && echo "$0: Wrong #accs" && exit 1; - $cmd $dir/log/update.$x.log \ - gmm-est --power=$power --write-occs=$dir/$[$x+1].occs --mix-up=$numgauss $dir/$x.mdl \ - "gmm-sum-accs - $dir/$x.*.acc |" $dir/$[$x+1].mdl || exit 1; - rm $dir/$x.mdl $dir/$x.*.acc - rm $dir/$x.occs - fi - [ $x -le $max_iter_inc ] && numgauss=$[$numgauss+$incgauss]; - x=$[$x+1]; -done - - -if [ $stage -le $x ]; then - # Accumulate stats for "alignment model"-- this model is - # computed with the speaker-independent features, but matches Gaussian-for-Gaussian - # with the final speaker-adapted model. - $cmd JOB=1:$nj $dir/log/acc_alimdl.JOB.log \ - ali-to-post "ark:gunzip -c $dir/ali.JOB.gz|" ark:- \| \ - gmm-acc-stats-twofeats $dir/$x.mdl "$feats" "$sifeats" \ - ark,s,cs:- $dir/$x.JOB.acc || exit 1; - [ `ls $dir/$x.*.acc | wc -w` -ne "$nj" ] && echo "$0: Wrong #accs" && exit 1; - # Update model. - $cmd $dir/log/est_alimdl.log \ - gmm-est --power=$power --remove-low-count-gaussians=false $dir/$x.mdl \ - "gmm-sum-accs - $dir/$x.*.acc|" $dir/$x.alimdl || exit 1; - rm $dir/$x.*.acc -fi - -rm $dir/final.{mdl,alimdl,occs} 2>/dev/null -ln -s $x.mdl $dir/final.mdl -ln -s $x.occs $dir/final.occs -ln -s $x.alimdl $dir/final.alimdl - - -steps/diagnostic/analyze_alignments.sh --cmd "$cmd" $lang $dir - -utils/summarize_warnings.pl $dir/log -( - echo "$0: Likelihood evolution:" - for x in `seq $[$num_iters-1]`; do - tail -n 30 $dir/log/acc.$x.*.log | awk '/Overall avg like/{l += $(NF-3)*$(NF-1); t += $(NF-1); } - /Overall average logdet/{d += $(NF-3)*$(NF-1); t2 += $(NF-1);} - END{ d /= t2; l /= t; printf("%s ", d+l); } ' - done - echo -) | tee $dir/log/summary.log - - -steps/info/gmm_dir_info.pl $dir - -echo "$0: done training SAT system in $dir" - -exit 0 diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/train.sh b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/train.sh deleted file mode 100644 index f3a3d3fc7cc98a38d8e9d523a0b43c0c8ea51bf9..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/kaldi_self_train/st/train.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash - -set -eu - -w2v_dir= # contains features `{train,valid}.{npy,lengths}`, real transcripts `{train,valid}.${label}`, and dict `dict.${label}.txt` -lab_dir= # contains pseudo labels `{train,valid}.txt` -out_dir= # output root -arpa_lm= # phone LM -arpa_lm_bin= # (binary) phone LM for KenLM, used in unsupervised selection - -label=phnc -train_name="train" -valid_name="valid" -data_dir=${out_dir}/data - -mkdir -p ${out_dir}/exp -local/prepare_lang.sh $w2v_dir/dict.${label}.txt $data_dir -local/prepare_lm.sh $arpa_lm $data_dir - -for x in $train_name $valid_name; do - x_gt=${x}_gt - - # prepare pseudo data - python local/prepare_data_from_w2v.py $w2v_dir $data_dir $x - steps/compute_cmvn_stats.sh $data_dir/$x $out_dir/exp/make_feat/$x $out_dir/feats/$x - python local/copy_aligned_text.py < $lab_dir/$x.txt > $data_dir/$x/text - - # prepare ground truth data - mkdir $data_dir/$x_gt - cp $data_dir/$x/{feats.scp,cmvn.scp,utt2spk,spk2utt} $data_dir/$x_gt/ - python local/copy_aligned_text.py < $w2v_dir/$x.$label > $data_dir/$x_gt/text -done - -local/train_subset_lgbeam.sh \ - --out_root ${out_dir} --out_name exp --train $train_name --valid $valid_name \ - --mono_size 2000 --tri1_size 5000 --tri2b_size -1 --tri3b_size -1 \ - --stage 1 --max_stage 3 $data_dir $data_dir/lang $data_dir/lang_test - -local/unsup_select_decode.sh \ - --split $valid_name --kenlm_path $arpa_lm_bin \ - --ref_txt $data_dir/${valid_name}_gt/text \ - --psd_txt $data_dir/${valid_name}/text \ - $out_dir/exp diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/models/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/models/__init__.py deleted file mode 100644 index 5b03ace727295f45f1fb268f56722491d6777d3f..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/models/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from .wav2vec_u import Wav2vec_U - -__all__ = [ - "Wav2vec_U", -] diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/models/wav2vec_u.py b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/models/wav2vec_u.py deleted file mode 100644 index 3f1232359796e529ea510101348446c22e4247c1..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/models/wav2vec_u.py +++ /dev/null @@ -1,636 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import math -from dataclasses import dataclass -from enum import Enum, auto -from typing import Dict, List, Optional, Tuple - -import numpy as np -import torch -import torch.nn as nn -import torch.nn.functional as F -from fairseq import checkpoint_utils, utils -from fairseq.dataclass import FairseqDataclass -from fairseq.models import BaseFairseqModel, register_model -from fairseq.modules import ( - SamePad, - TransposeLast, -) -from torch import autograd - - -class SegmentationType(Enum): - NONE = auto() - RANDOM = auto() - UNIFORM_RANDOM = auto() - UNIFORM_RANDOM_JOIN = auto() - JOIN = auto() - - -@dataclass -class SegmentationConfig(FairseqDataclass): - type: SegmentationType = SegmentationType.NONE - subsample_rate: float = 0.25 - mean_pool: bool = True - mean_pool_join: bool = False - remove_zeros: bool = False - - -@dataclass -class Wav2vec_UConfig(FairseqDataclass): - - discriminator_kernel: int = 3 - discriminator_dilation: int = 1 - discriminator_dim: int = 256 - discriminator_causal: bool = True - discriminator_linear_emb: bool = False - discriminator_depth: int = 1 - discriminator_max_pool: bool = False - discriminator_act_after_linear: bool = False - discriminator_dropout: float = 0.0 - discriminator_spectral_norm: bool = False - discriminator_weight_norm: bool = False - - generator_kernel: int = 4 - generator_dilation: int = 1 - generator_stride: int = 1 - generator_bias: bool = False - generator_dropout: float = 0.0 - - blank_weight: float = 0 - blank_mode: str = "add" - blank_is_sil: bool = False - no_softmax: bool = False - - smoothness_weight: float = 0.0 - smoothing: float = 0.0 - smoothing_one_sided: bool = False - gradient_penalty: float = 0.0 - probabilistic_grad_penalty_slicing: bool = False - code_penalty: float = 0.0 - gumbel: bool = False - hard_gumbel: bool = True - temp: Tuple[float, float, float] = (2, 0.1, 0.99995) - input_dim: int = 128 - - segmentation: SegmentationConfig = SegmentationConfig() - - -class Segmenter(nn.Module): - cfg: SegmentationConfig - - def __init__(self, cfg: SegmentationConfig): - super().__init__() - self.cfg = cfg - self.subsample_rate = cfg.subsample_rate - - def pre_segment(self, dense_x, dense_padding_mask): - return dense_x, dense_padding_mask - - def logit_segment(self, logits, padding_mask): - return logits, padding_mask - - -class RandomSegmenter(Segmenter): - def pre_segment(self, dense_x, dense_padding_mask): - target_num = math.ceil(dense_x.size(1) * self.subsample_rate) - ones = torch.ones(dense_x.shape[:-1], device=dense_x.device) - indices, _ = ones.multinomial(target_num).sort(dim=-1) - indices_ld = indices.unsqueeze(-1).expand(-1, -1, dense_x.size(-1)) - dense_x = dense_x.gather(1, indices_ld) - dense_padding_mask = dense_padding_mask.gather(1, index=indices) - return dense_x, dense_padding_mask - - -class UniformRandomSegmenter(Segmenter): - def pre_segment(self, dense_x, dense_padding_mask): - bsz, tsz, fsz = dense_x.shape - - target_num = math.ceil(tsz * self.subsample_rate) - - rem = tsz % target_num - - if rem > 0: - dense_x = F.pad(dense_x, [0, 0, 0, target_num - rem]) - dense_padding_mask = F.pad( - dense_padding_mask, [0, target_num - rem], value=True - ) - - dense_x = dense_x.view(bsz, target_num, -1, fsz) - dense_padding_mask = dense_padding_mask.view(bsz, target_num, -1) - - if self.cfg.mean_pool: - dense_x = dense_x.mean(dim=-2) - dense_padding_mask = dense_padding_mask.all(dim=-1) - else: - ones = torch.ones((bsz, dense_x.size(2)), device=dense_x.device) - indices = ones.multinomial(1) - indices = indices.unsqueeze(-1).expand(-1, target_num, -1) - indices_ld = indices.unsqueeze(-1).expand(-1, -1, -1, fsz) - dense_x = dense_x.gather(2, indices_ld).reshape(bsz, -1, fsz) - dense_padding_mask = dense_padding_mask.gather(2, index=indices).reshape( - bsz, -1 - ) - return dense_x, dense_padding_mask - - -class JoinSegmenter(Segmenter): - def logit_segment(self, logits, padding_mask): - preds = logits.argmax(dim=-1) - - if padding_mask.any(): - preds[padding_mask] = -1 # mark pad - uniques = [] - - bsz, tsz, csz = logits.shape - - for p in preds: - uniques.append( - p.cpu().unique_consecutive(return_inverse=True, return_counts=True) - ) - - new_tsz = max(u[0].numel() for u in uniques) - new_logits = logits.new_zeros(bsz, new_tsz, csz) - new_pad = padding_mask.new_zeros(bsz, new_tsz) - - for b in range(bsz): - u, idx, c = uniques[b] - keep = u != -1 - - if self.cfg.remove_zeros: - keep.logical_and_(u != 0) - - if self.training and not self.cfg.mean_pool_join: - u[0] = 0 - u[1:] = c.cumsum(0)[:-1] - m = c > 1 - r = torch.rand(m.sum()) - o = (c[m] * r).long() - u[m] += o - new_logits[b, : u.numel()] = logits[b, u] - else: - new_logits[b].index_add_( - dim=0, index=idx.to(new_logits.device), source=logits[b] - ) - new_logits[b, : c.numel()] /= c.unsqueeze(-1).to(new_logits.device) - - new_sz = keep.sum() - if not keep.all(): - kept_logits = new_logits[b, : c.numel()][keep] - new_logits[b, :new_sz] = kept_logits - - if new_sz < new_tsz: - pad = new_tsz - new_sz - new_logits[b, -pad:] = 0 - new_pad[b, -pad:] = True - - return new_logits, new_pad - - -class UniformRandomJoinSegmenter(UniformRandomSegmenter, JoinSegmenter): - pass - - -SEGMENT_FACTORY = { - SegmentationType.NONE: Segmenter, - SegmentationType.RANDOM: RandomSegmenter, - SegmentationType.UNIFORM_RANDOM: UniformRandomSegmenter, - SegmentationType.UNIFORM_RANDOM_JOIN: UniformRandomJoinSegmenter, - SegmentationType.JOIN: JoinSegmenter, -} - - -class Discriminator(nn.Module): - def __init__(self, dim, cfg: Wav2vec_UConfig): - super().__init__() - - inner_dim = cfg.discriminator_dim - kernel = cfg.discriminator_kernel - dilation = cfg.discriminator_dilation - self.max_pool = cfg.discriminator_max_pool - - if cfg.discriminator_causal: - padding = kernel - 1 - else: - padding = kernel // 2 - - def make_conv(in_d, out_d, k, p=0, has_dilation=True): - conv = nn.Conv1d( - in_d, - out_d, - kernel_size=k, - padding=p, - dilation=dilation if has_dilation else 1, - ) - if cfg.discriminator_spectral_norm: - conv = nn.utils.spectral_norm(conv) - elif cfg.discriminator_weight_norm: - conv = nn.utils.weight_norm(conv) - return conv - - inner_net = [ - nn.Sequential( - make_conv(inner_dim, inner_dim, kernel, padding), - SamePad(kernel_size=kernel, causal=cfg.discriminator_causal), - nn.Dropout(cfg.discriminator_dropout), - nn.GELU(), - ) - for _ in range(cfg.discriminator_depth - 1) - ] + [ - make_conv(inner_dim, 1, kernel, padding, has_dilation=False), - SamePad(kernel_size=kernel, causal=cfg.discriminator_causal), - ] - - if cfg.discriminator_linear_emb: - emb_net = [make_conv(dim, inner_dim, 1)] - else: - emb_net = [ - make_conv(dim, inner_dim, kernel, padding), - SamePad(kernel_size=kernel, causal=cfg.discriminator_causal), - ] - - if cfg.discriminator_act_after_linear: - emb_net.append(nn.GELU()) - - self.net = nn.Sequential( - *emb_net, - nn.Dropout(cfg.discriminator_dropout), - *inner_net, - ) - - def forward(self, x, padding_mask): - x = x.transpose(1, 2) # BTC -> BCT - x = self.net(x) - x = x.transpose(1, 2) - x_sz = x.size(1) - if padding_mask is not None and padding_mask.any() and padding_mask.dim() > 1: - padding_mask = padding_mask[:, : x.size(1)] - x[padding_mask] = float("-inf") if self.max_pool else 0 - x_sz = x_sz - padding_mask.sum(dim=-1) - x = x.squeeze(-1) - if self.max_pool: - x, _ = x.max(dim=-1) - else: - x = x.sum(dim=-1) - x = x / x_sz - return x - - -class Generator(nn.Module): - def __init__(self, input_dim, output_dim, cfg: Wav2vec_UConfig): - super().__init__() - - self.cfg = cfg - self.output_dim = output_dim - self.stride = cfg.generator_stride - self.dropout = nn.Dropout(cfg.generator_dropout) - - padding = cfg.generator_kernel // 2 - self.proj = nn.Sequential( - TransposeLast(), - nn.Conv1d( - input_dim, - output_dim, - kernel_size=cfg.generator_kernel, - stride=cfg.generator_stride, - dilation=cfg.generator_dilation, - padding=padding, - bias=cfg.generator_bias, - ), - TransposeLast(), - ) - - def forward(self, dense_x, tokens, dense_padding_mask): - dense_x = self.dropout(dense_x) - - dense_x = self.proj(dense_x) - if self.stride > 1: - dense_padding_mask = dense_padding_mask[:, :: self.stride] - - if dense_padding_mask.size(1) != dense_x.size(1): - new_padding = dense_padding_mask.new_zeros(dense_x.shape[:-1]) - diff = new_padding.size(1) - dense_padding_mask.size(1) - assert ( - diff > 0 - ), f"{new_padding.shape}, {dense_padding_mask.shape}, {dense_x.shape}, {diff}" - if diff > 0: - new_padding[:, diff:] = dense_padding_mask - else: - assert diff < 0 - new_padding = dense_padding_mask[:, :diff] - - dense_padding_mask = new_padding - - result = {} - - token_x = None - if tokens is not None: - token_x = dense_x.new_zeros(tokens.numel(), self.output_dim) - token_x.scatter_(1, tokens.view(-1, 1).long(), 1) - token_x = token_x.view(tokens.shape + (self.output_dim,)) - - result["dense_x"] = dense_x - result["token_x"] = token_x - result["dense_padding_mask"] = dense_padding_mask - - return result - - -@register_model("wav2vec_u", dataclass=Wav2vec_UConfig) -class Wav2vec_U(BaseFairseqModel): - def calc_gradient_penalty(self, real_data, fake_data): - - b_size = min(real_data.size(0), fake_data.size(0)) - t_size = min(real_data.size(1), fake_data.size(1)) - - if self.cfg.probabilistic_grad_penalty_slicing: - - def get_slice(data, dim, target_size): - - size = data.size(dim) - diff = size - target_size - if diff <= 0: - return data - - start = np.random.randint(0, diff + 1) - return data.narrow(dim=dim, start=start, length=target_size) - - real_data = get_slice(real_data, 0, b_size) - real_data = get_slice(real_data, 1, t_size) - fake_data = get_slice(fake_data, 0, b_size) - fake_data = get_slice(fake_data, 1, t_size) - - else: - real_data = real_data[:b_size, :t_size] - fake_data = fake_data[:b_size, :t_size] - - alpha = torch.rand(real_data.size(0), 1, 1) - alpha = alpha.expand(real_data.size()) - alpha = alpha.to(real_data.device) - - interpolates = alpha * real_data + ((1 - alpha) * fake_data) - - disc_interpolates = self.discriminator(interpolates, None) - - gradients = autograd.grad( - outputs=disc_interpolates, - inputs=interpolates, - grad_outputs=torch.ones(disc_interpolates.size(), device=real_data.device), - create_graph=True, - retain_graph=True, - only_inputs=True, - )[0] - - gradient_penalty = (gradients.norm(2, dim=1) - 1) ** 2 - return gradient_penalty - - def set_num_updates(self, num_updates): - super().set_num_updates(num_updates) - self.update_num = num_updates - self.curr_temp = max( - self.max_temp * self.temp_decay ** num_updates, self.min_temp - ) - - def discrim_step(self, num_updates): - return num_updates % 2 == 1 - - def get_groups_for_update(self, num_updates): - return "discriminator" if self.discrim_step(num_updates) else "generator" - - def __init__(self, cfg: Wav2vec_UConfig, target_dict): - super().__init__() - - self.cfg = cfg - self.zero_index = target_dict.index("<SIL>") if "<SIL>" in target_dict else 0 - self.smoothness_weight = cfg.smoothness_weight - - output_size = len(target_dict) - self.pad = target_dict.pad() - self.eos = target_dict.eos() - self.smoothing = cfg.smoothing - self.smoothing_one_sided = cfg.smoothing_one_sided - self.no_softmax = cfg.no_softmax - self.gumbel = cfg.gumbel - self.hard_gumbel = cfg.hard_gumbel - self.last_acc = None - - self.gradient_penalty = cfg.gradient_penalty - self.code_penalty = cfg.code_penalty - self.blank_weight = cfg.blank_weight - self.blank_mode = cfg.blank_mode - self.blank_index = target_dict.index("<SIL>") if cfg.blank_is_sil else 0 - assert self.blank_index != target_dict.unk() - - self.discriminator = Discriminator(output_size, cfg) - for p in self.discriminator.parameters(): - p.param_group = "discriminator" - - self.pca_A = self.pca_b = None - d = cfg.input_dim - - self.segmenter = SEGMENT_FACTORY[cfg.segmentation.type](cfg.segmentation) - - self.generator = Generator(d, output_size, cfg) - - for p in self.generator.parameters(): - p.param_group = "generator" - - for p in self.segmenter.parameters(): - p.param_group = "generator" - - self.max_temp, self.min_temp, self.temp_decay = cfg.temp - self.curr_temp = self.max_temp - self.update_num = 0 - - @classmethod - def build_model(cls, cfg, task): - return cls(cfg, task.target_dictionary) - - def get_logits( - self, - net_output: Optional[Dict[str, List[Optional[torch.Tensor]]]], - normalize: bool = False, - ): - logits = net_output["logits"] - - if self.blank_weight != 0: - if self.blank_mode == "add": - logits[..., self.blank_index] += self.blank_weight - elif self.blank_mode == "set": - logits[..., self.blank_index] = self.blank_weight - else: - raise Exception(f"invalid blank mode {self.blank_mode}") - - padding = net_output["padding_mask"] - if padding.any(): - logits[padding] = float("-inf") - logits[padding][..., self.blank_index] = float("inf") - - if normalize: - logits = utils.log_softmax(logits.float(), dim=-1) - - return logits.transpose(0, 1) - - def get_normalized_probs( - self, - net_output: Tuple[ - torch.Tensor, Optional[Dict[str, List[Optional[torch.Tensor]]]] - ], - log_probs: bool, - sample: Optional[Dict[str, torch.Tensor]] = None, - ): - logits = self.get_logits(net_output) - - probs = super().get_normalized_probs(logits, log_probs, sample) - # BTC -> TBC for ctc - probs = probs.transpose(0, 1) - return probs - - def normalize(self, dense_x): - - bsz, tsz, csz = dense_x.shape - - if dense_x.numel() == 0: - raise Exception(dense_x.shape) - _, k = dense_x.max(-1) - hard_x = ( - dense_x.new_zeros(bsz * tsz, csz) - .scatter_(-1, k.view(-1, 1), 1.0) - .view(-1, csz) - ) - hard_probs = torch.mean(hard_x.float(), dim=0) - code_perplexity = torch.exp( - -torch.sum(hard_probs * torch.log(hard_probs + 1e-7), dim=-1) - ) - - avg_probs = torch.softmax(dense_x.reshape(-1, csz).float(), dim=-1).mean(dim=0) - prob_perplexity = torch.exp( - -torch.sum(avg_probs * torch.log(avg_probs + 1e-7), dim=-1) - ) - - if not self.no_softmax: - if self.training and self.gumbel: - dense_x = F.gumbel_softmax( - dense_x.float(), tau=self.curr_temp, hard=self.hard_gumbel - ).type_as(dense_x) - else: - dense_x = dense_x.softmax(-1) - - return dense_x, code_perplexity, prob_perplexity - - def forward( - self, - features, - padding_mask, - random_label=None, - dense_x_only=False, - segment=True, - ): - if segment: - features, padding_mask = self.segmenter.pre_segment(features, padding_mask) - - orig_size = features.size(0) * features.size(1) - padding_mask.sum() - - gen_result = self.generator(features, random_label, padding_mask) - - orig_dense_x, token_x = gen_result["dense_x"], gen_result["token_x"] - orig_dense_padding_mask = gen_result["dense_padding_mask"] - - if segment: - dense_x, dense_padding_mask = self.segmenter.logit_segment( - orig_dense_x, orig_dense_padding_mask - ) - else: - dense_x = orig_dense_x - dense_padding_mask = orig_dense_padding_mask - - dense_logits = dense_x - prob_perplexity = None - code_perplexity = None - - if not (self.no_softmax and dense_x_only): - dense_x, code_perplexity, prob_perplexity = self.normalize(dense_logits) - - if dense_x_only or self.discriminator is None: - return { - "logits": dense_x, - "padding_mask": dense_padding_mask, - } - - token_padding_mask = random_label == self.pad - - dense_y = self.discriminator(dense_x, dense_padding_mask) - token_y = self.discriminator(token_x, token_padding_mask) - - sample_size = features.size(0) - - d_step = self.discrim_step(self.update_num) - - fake_smooth = self.smoothing - real_smooth = self.smoothing - if self.smoothing_one_sided: - fake_smooth = 0 - - zero_loss = None - smoothness_loss = None - code_pen = None - - if d_step: - loss_dense = F.binary_cross_entropy_with_logits( - dense_y, - dense_y.new_ones(dense_y.shape) - fake_smooth, - reduction="sum", - ) - loss_token = F.binary_cross_entropy_with_logits( - token_y, - token_y.new_zeros(token_y.shape) + real_smooth, - reduction="sum", - ) - if self.training and self.gradient_penalty > 0: - grad_pen = self.calc_gradient_penalty(token_x, dense_x) - grad_pen = grad_pen.sum() * self.gradient_penalty - else: - grad_pen = None - else: - grad_pen = None - loss_token = None - loss_dense = F.binary_cross_entropy_with_logits( - dense_y, - dense_y.new_zeros(dense_y.shape) + fake_smooth, - reduction="sum", - ) - num_vars = dense_x.size(-1) - if prob_perplexity is not None: - code_pen = (num_vars - prob_perplexity) / num_vars - code_pen = code_pen * sample_size * self.code_penalty - - if self.smoothness_weight > 0: - smoothness_loss = F.mse_loss( - dense_logits[:, :-1], dense_logits[:, 1:], reduction="none" - ) - smoothness_loss[dense_padding_mask[:, 1:]] = 0 - smoothness_loss = ( - smoothness_loss.mean() * sample_size * self.smoothness_weight - ) - - result = { - "losses": { - "grad_pen": grad_pen, - "code_pen": code_pen, - "smoothness": smoothness_loss, - }, - "temp": self.curr_temp, - "code_ppl": code_perplexity, - "prob_ppl": prob_perplexity, - "d_steps": int(d_step), - "sample_size": sample_size, - } - - suff = "_d" if d_step else "_g" - result["losses"]["dense" + suff] = loss_dense - result["losses"]["token" + suff] = loss_token - - return result diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/apply_pca.py b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/apply_pca.py deleted file mode 100644 index 4f035f35023aa44fca237043fd4ff06c36e571a4..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/apply_pca.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env python3 -u -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import math -import os -import os.path as osp -from shutil import copyfile - -import numpy as np -import torch -import tqdm -from npy_append_array import NpyAppendArray - - -def get_parser(): - parser = argparse.ArgumentParser( - description="transforms features via a given pca and stored them in target dir" - ) - # fmt: off - parser.add_argument('source', help='directory with features') - parser.add_argument('--split', help='which split to read', required=True) - parser.add_argument('--save-dir', help='where to save the output', required=True) - parser.add_argument('--pca-path', type=str, help='pca location. will append _A.npy and _b.npy', required=True) - parser.add_argument('--batch-size', type=int, default=2048000, help='batch size') - parser.add_argument('--unfiltered', action='store_true', help='process the unfiltered version') - # fmt: on - - return parser - - -def main(): - parser = get_parser() - args = parser.parse_args() - - source_path = osp.join(args.source, args.split) - data_poth = source_path + "_unfiltered" if args.unfiltered else source_path - - print(f"data path: {data_poth}") - - features = np.load(data_poth + ".npy", mmap_mode="r") - pca_A = torch.from_numpy(np.load(args.pca_path + "_A.npy")).cuda() - pca_b = torch.from_numpy(np.load(args.pca_path + "_b.npy")).cuda() - - os.makedirs(args.save_dir, exist_ok=True) - save_path = osp.join(args.save_dir, args.split) - - copyfile(source_path + ".tsv", save_path + ".tsv") - copyfile(data_poth + ".lengths", save_path + ".lengths") - - if osp.exists(source_path + ".phn"): - copyfile(source_path + ".phn", save_path + ".phn") - - if osp.exists(source_path + ".wrd"): - copyfile(source_path + ".wrd", save_path + ".wrd") - - if osp.exists(save_path + ".npy"): - os.remove(save_path + ".npy") - npaa = NpyAppendArray(save_path + ".npy") - - batches = math.ceil(features.shape[0] / args.batch_size) - - with torch.no_grad(): - for b in tqdm.trange(batches): - start = b * args.batch_size - end = start + args.batch_size - x = torch.from_numpy(features[start:end]).cuda() - x = torch.matmul(x, pca_A) + pca_b - npaa.append(x.cpu().numpy()) - - -if __name__ == "__main__": - main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/copy_labels.py b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/copy_labels.py deleted file mode 100644 index 989868388eefccc37c82d7602f709632035c7aa1..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/copy_labels.py +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env python3 -u -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import sys - -for idx, line in enumerate(sys.stdin): - print(f"utt{idx:010d} {line}", end="") diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/filter_lexicon.py b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/filter_lexicon.py deleted file mode 100644 index 5bf3e51e7a50ac3f07cc41739198cde946dc79aa..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/filter_lexicon.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python3 -u -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import sys - -from fairseq.data import Dictionary - - -def get_parser(): - parser = argparse.ArgumentParser( - description="filters a lexicon given a unit dictionary" - ) - parser.add_argument("-d", "--unit-dict", help="unit dictionary", required=True) - return parser - - -def main(): - parser = get_parser() - args = parser.parse_args() - - d = Dictionary.load(args.unit_dict) - symbols = set(d.symbols) - - for line in sys.stdin: - items = line.rstrip().split() - skip = len(items) < 2 - for x in items[1:]: - if x not in symbols: - skip = True - break - if not skip: - print(line, end="") - - -if __name__ == "__main__": - main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/filter_tsv.py b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/filter_tsv.py deleted file mode 100644 index 538207e989346daadb34511bea884e5e749c82e9..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/filter_tsv.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python3 -u -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import os -import sys - -parser = argparse.ArgumentParser() -parser.add_argument("--tsv", required=True, type=str) -parser.add_argument("--no-skip", action="store_true") -parser.add_argument("--keep", action="store_true") -params = parser.parse_args() - - -def get_fname(line): - p = os.path.basename(line.split("\t")[0]) - p = os.path.splitext(p)[0] - return p - - -# filenames to exclude -seen = set() -with open(params.tsv) as f: - if not params.no_skip: - root = next(f).rstrip() - for line in f: - seen.add(get_fname(line)) - -for i, line in enumerate(sys.stdin): - exists = get_fname(line) in seen - keep = (exists and params.keep) or (not exists and not params.keep) - if i == 0 or keep: - print(line, end="") diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/g2p_wrd_to_phn.py b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/g2p_wrd_to_phn.py deleted file mode 100644 index 2e31c307bd67d10941150160c7fb8c9e085ac5d9..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/g2p_wrd_to_phn.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python3 -u -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import sys - -from g2p_en import G2p - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument( - "--compact", - action="store_true", - help="if set, compacts phones", - ) - args = parser.parse_args() - - compact = args.compact - - wrd_to_phn = {} - g2p = G2p() - for line in sys.stdin: - words = line.strip().split() - phones = [] - for w in words: - if w not in wrd_to_phn: - wrd_to_phn[w] = g2p(w) - if compact: - wrd_to_phn[w] = [ - p[:-1] if p[-1].isnumeric() else p for p in wrd_to_phn[w] - ] - phones.extend(wrd_to_phn[w]) - try: - print(" ".join(phones)) - except: - print(wrd_to_phn, words, phones, file=sys.stderr) - raise - - -if __name__ == "__main__": - main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/ltr_to_wrd.py b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/ltr_to_wrd.py deleted file mode 100644 index 36c85d1e2f60487494a92207feb4685e78db8aa2..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/ltr_to_wrd.py +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env python3 -u -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import sys - - -def main(): - for line in sys.stdin: - print(line.replace(" ", "").replace("|", " ").strip()) - - -if __name__ == "__main__": - main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/mean_pool.py b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/mean_pool.py deleted file mode 100644 index ee6172040508672f5dc4eee0ebe5167a4c305364..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/mean_pool.py +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env python3 -u -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import math -import os -import os.path as osp -from shutil import copyfile - -import numpy as np -import torch -import torch.nn.functional as F -import tqdm -from npy_append_array import NpyAppendArray - - -def get_parser(): - parser = argparse.ArgumentParser( - description="mean pools representations by compressing uniform splits of the data" - ) - # fmt: off - parser.add_argument('source', help='directory with features') - parser.add_argument('--split', help='which split to read', required=True) - parser.add_argument('--save-dir', help='where to save the output', required=True) - parser.add_argument('--subsample-rate', type=float, default=0.5, help='size to subsample data to') - - parser.add_argument('--remove-extra', action='store_true', help='if true, removes extra states that cant be pooled, otherwise pads with 0s') - # fmt: on - - return parser - - -def main(): - parser = get_parser() - args = parser.parse_args() - - source_path = osp.join(args.source, args.split) - - print(f"data path: {source_path}") - - features = np.load(source_path + ".npy", mmap_mode="r") - - os.makedirs(args.save_dir, exist_ok=True) - save_path = osp.join(args.save_dir, args.split) - - copyfile(source_path + ".tsv", save_path + ".tsv") - - if os.path.exists(source_path + ".phn"): - copyfile(source_path + ".phn", save_path + ".phn") - if os.path.exists(source_path + ".wrd"): - copyfile(source_path + ".wrd", save_path + ".wrd") - - if os.path.exists(osp.join(args.source, "dict.phn.txt")): - copyfile( - osp.join(args.source, "dict.phn.txt"), - osp.join(args.save_dir, "dict.phn.txt"), - ) - - if osp.exists(save_path + ".npy"): - os.remove(save_path + ".npy") - npaa = NpyAppendArray(save_path + ".npy") - - with open(source_path + ".lengths", "r") as lf: - lengths = lf.readlines() - - fsz = features.shape[-1] - start = 0 - with torch.no_grad(): - with open(save_path + ".lengths", "w") as lengths_out: - for length in tqdm.tqdm(lengths): - length = int(length) - end = start + length - feats = features[start:end] - start += length - x = torch.from_numpy(feats).cuda() - target_num = math.ceil(length * args.subsample_rate) - rem = length % target_num - - if rem > 0: - if args.remove_extra: - to_rem = target_num - rem - target_num -= 1 - x = x[:-to_rem] - else: - to_add = target_num - rem - x = F.pad(x, [0, 0, 0, to_add]) - x[-to_add:] = x[-to_add - 1] - - x = x.view(target_num, -1, fsz) - x = x.mean(dim=-2) - print(target_num, file=lengths_out) - npaa.append(x.cpu().numpy()) - - -if __name__ == "__main__": - main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/merge_clusters.py b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/merge_clusters.py deleted file mode 100644 index 089f50589191399c17b7b722084a2473cf86029a..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/merge_clusters.py +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env python3 -u -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import os -import os.path as osp -import random -from shutil import copyfile - -import numpy as np -import torch -import tqdm -from npy_append_array import NpyAppendArray - - -def get_parser(): - parser = argparse.ArgumentParser( - description="transforms features via a given pca and stored them in target dir" - ) - # fmt: off - parser.add_argument('source', help='directory with features') - parser.add_argument('--split', help='which split to read', required=True) - parser.add_argument('--save-dir', help='where to save the output', required=True) - parser.add_argument('--cluster-dir', help='where the clusters are') - parser.add_argument('--pooling', type=str, default='mean', choices=['mean', 'sample'], help='how to pool') - # fmt: on - - return parser - - -def main(): - parser = get_parser() - args = parser.parse_args() - - source_path = osp.join(args.source, args.split) - cluster_path = osp.join(args.cluster_dir, args.split + ".src") - print(f"data path: {source_path}") - - features = np.load(source_path + ".npy", mmap_mode="r") - sizes = [] - offsets = [] - offset = 0 - with open(source_path + ".lengths", "r") as len_f: - for line in len_f: - length = int(line.rstrip()) - sizes.append(length) - offsets.append(offset) - offset += length - - clusters = [] - with open(cluster_path, "r") as cf: - for line in cf: - line = line.rstrip() - items = line.split() - items = list(map(int, items)) - clusters.append(items) - - os.makedirs(args.save_dir, exist_ok=True) - save_path = osp.join(args.save_dir, args.split) - - copyfile(source_path + ".tsv", save_path + ".tsv") - - if os.path.exists(source_path + ".phn"): - copyfile(source_path + ".phn", save_path + ".phn") - if os.path.exists(osp.join(args.source, "dict.phn.txt")): - copyfile( - osp.join(args.source, "dict.phn.txt"), - osp.join(args.save_dir, "dict.phn.txt"), - ) - if os.path.exists(source_path + ".wrd"): - copyfile(source_path + ".wrd", save_path + ".wrd") - - if osp.exists(save_path + ".npy"): - os.remove(save_path + ".npy") - npaa = NpyAppendArray(save_path + ".npy") - - def merge(feats, clust): - feats = torch.from_numpy(feats.copy()) - clust = torch.LongTensor(clust) - _, counts = clust.unique_consecutive(return_counts=True) - curr = 0 - - merged = [] - for c in counts: - c = c.item() - start = curr - end = curr + c - curr += c - if args.pooling == "mean": - new_x = feats[start:end].mean(dim=0) - elif args.pooling == "sample": - new_x = feats[start + int(random.random() * c)] - else: - raise NotImplementedError() - merged.append(new_x) - - return torch.stack(merged, dim=0).numpy() - - with open(save_path + ".lengths", "w") as l_f: - for size, offset, clust in tqdm.tqdm( - zip(sizes, offsets, clusters), total=len(sizes) - ): - end = size + offset - feats = features[offset:end] - feats = merge(feats, clust) - print(len(feats), file=l_f) - npaa.append(feats) - - -if __name__ == "__main__": - main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/normalize_and_filter_text.py b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/normalize_and_filter_text.py deleted file mode 100644 index 9c3050ac778c4434b7189229ad668ac32d776779..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/normalize_and_filter_text.py +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import os -import sys - -import fasttext as ft -import regex - - -def get_parser(): - parser = argparse.ArgumentParser( - description="reads text from stdin and outputs normalized, lid-filtered version to stdout" - ) - parser.add_argument( - "--fasttext-model", - help="path to fasttext model", - default="lid.187.bin", - ) - parser.add_argument("--lang", help="language id", required=True) - parser.add_argument( - "--lid-threshold", - type=float, - help="threshold for this lang id probability", - default=0.4, - ) - - return parser - - -def main(): - parser = get_parser() - args = parser.parse_args() - filter_r = regex.compile(r"[^\p{L}\p{N}\p{M}\' \-]") - - lg = args.lang.lower() - lg_label = f"__label__{lg}" - thresh = args.lid_threshold - - if os.path.exists(args.fasttext_model): - model = ft.load_model(args.fasttext_model) - else: - print( - f"fasttext language id model {args.fasttext_model} not found. Proceeding without language filtering. " - f"To enable language filtering, please download the latest language id model " - f"from https://fasttext.cc/docs/en/language-identification.html", - file=sys.stderr, - ) - model = None - - for line in sys.stdin: - line = line.strip() - line = filter_r.sub(" ", line) - line = " ".join(line.split()) - - if model is not None: - lid, prob = model.predict(line, k=100) - try: - target_idx = lid.index(lg_label) - except ValueError: - continue - if target_idx == 0 or prob[target_idx] >= thresh: - print(line) - else: - print(line) - - -if __name__ == "__main__": - main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/normalize_text.py b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/normalize_text.py deleted file mode 100644 index 534ea49f05ce32ef33bfc2dbb3441bc50a5f0c62..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/normalize_text.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import sys - -import regex - - -def main(): - filter_r = regex.compile(r"[^\p{L}\p{N}\p{M}\' \-]") - - for line in sys.stdin: - line = line.strip() - line = filter_r.sub(" ", line) - line = " ".join(line.split()) - print(line) - - -if __name__ == "__main__": - main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/pca.py b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/pca.py deleted file mode 100644 index 047935206c1574744abe751b0f932d170cbe8f8a..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/pca.py +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env python3 -u -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import os -import os.path as osp - -import faiss -import numpy as np - - -def get_parser(): - parser = argparse.ArgumentParser( - description="compute a pca matrix given an array of numpy features" - ) - # fmt: off - parser.add_argument('data', help='numpy file containing features') - parser.add_argument('--output', help='where to save the pca matrix', required=True) - parser.add_argument('--dim', type=int, help='dim for pca reduction', required=True) - parser.add_argument('--eigen-power', type=float, default=0, help='eigen power, -0.5 for whitening') - - return parser - - -def main(): - parser = get_parser() - args = parser.parse_args() - - print("Reading features") - x = np.load(args.data, mmap_mode="r") - - print("Computing PCA") - pca = faiss.PCAMatrix(x.shape[-1], args.dim, args.eigen_power) - pca.train(x) - b = faiss.vector_to_array(pca.b) - A = faiss.vector_to_array(pca.A).reshape(pca.d_out, pca.d_in) - - os.makedirs(args.output, exist_ok=True) - - prefix = str(args.dim) - if args.eigen_power != 0: - prefix += f"_{args.eigen_power}" - - np.save(osp.join(args.output, f"{prefix}_pca_A"), A.T) - np.save(osp.join(args.output, f"{prefix}_pca_b"), b) - - -if __name__ == "__main__": - main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/phonemize_with_sil.py b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/phonemize_with_sil.py deleted file mode 100644 index 04600c9f58a5f0dfe4f2b85510800d2dc25257e6..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/phonemize_with_sil.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python3 -u -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import sys - -import numpy as np - - -def get_parser(): - parser = argparse.ArgumentParser( - description="converts words to phones adding optional silences around in between words" - ) - parser.add_argument( - "--sil-prob", - "-s", - type=float, - default=0, - help="probability of inserting silence between each word", - ) - parser.add_argument( - "--surround", - action="store_true", - help="if set, surrounds each example with silence", - ) - parser.add_argument( - "--lexicon", - help="lexicon to convert to phones", - required=True, - ) - - return parser - - -def main(): - parser = get_parser() - args = parser.parse_args() - - sil_prob = args.sil_prob - surround = args.surround - sil = "<SIL>" - - wrd_to_phn = {} - - with open(args.lexicon, "r") as lf: - for line in lf: - items = line.rstrip().split() - assert len(items) > 1, line - assert items[0] not in wrd_to_phn, items - wrd_to_phn[items[0]] = items[1:] - - for line in sys.stdin: - words = line.strip().split() - - if not all(w in wrd_to_phn for w in words): - continue - - phones = [] - if surround: - phones.append(sil) - - sample_sil_probs = None - if sil_prob > 0 and len(words) > 1: - sample_sil_probs = np.random.random(len(words) - 1) - - for i, w in enumerate(words): - phones.extend(wrd_to_phn[w]) - if ( - sample_sil_probs is not None - and i < len(sample_sil_probs) - and sample_sil_probs[i] < sil_prob - ): - phones.append(sil) - - if surround: - phones.append(sil) - print(" ".join(phones)) - - -if __name__ == "__main__": - main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/prepare_audio.sh b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/prepare_audio.sh deleted file mode 100644 index 013f7a9b055a7693a29f9c5ba1e4003a9a25850e..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/prepare_audio.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env zsh -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -source_dir=$1 -tgt_dir=$2 -model=$3 - -if [ -z "$4" ] - then - dim=512 - else - dim=$4 -fi - -echo "using $dim dim for PCA" - -if [ -z "$5" ] - then - layer=14 - else - layer=$5 -fi - -echo "extracting from layer $layer" - -train_split=train -valid_split=valid -test_split=test - -all_splits=($train_split) - -if [[ -f "$source_dir/valid.tsv" ]]; then - all_splits+=('valid') -fi - -if [[ -f "$source_dir/test.tsv" ]]; then - all_splits+=('test') -fi - -echo "processing splits: $all_splits" - -mkdir -p $tgt_dir - -cp $source_dir/*.tsv $tgt_dir -cp $source_dir/*.wrd $tgt_dir -cp $source_dir/*.ltr $tgt_dir -cp $source_dir/*.phn $tgt_dir -cp $source_dir/dict* $tgt_dir - -setopt shwordsplit - -for split in $all_splits; do - python $FAIRSEQ_ROOT/examples/wav2vec/unsupervised/scripts/wav2vec_extract_features.py $source_dir --split $split \ - --save-dir $tgt_dir --checkpoint $model --layer $layer -done - -python $FAIRSEQ_ROOT/examples/wav2vec/unsupervised/scripts/wav2vec_cluster_faiss.py $tgt_dir/${train_split}.tsv \ ---checkpoint $model --save-dir $tgt_dir -f "CLUS128" --sample-pct 1.0 - -for split in $all_splits; do - python $FAIRSEQ_ROOT/examples/wav2vec/unsupervised/scripts/wav2vec_apply_cluster_faiss.py $tgt_dir \ - --checkpoint $model --path $tgt_dir/CLUS128 --split $split -done - -python $FAIRSEQ_ROOT/examples/wav2vec/unsupervised/scripts/pca.py $tgt_dir/${train_split}.npy --output $tgt_dir/pca --dim $dim - -for split in $all_splits; do - python $FAIRSEQ_ROOT/examples/wav2vec/unsupervised/scripts/apply_pca.py $tgt_dir --split $split --save-dir $tgt_dir/precompute_pca$dim --pca-path $tgt_dir/pca/${dim}_pca --batch-size 1048000 - - python $FAIRSEQ_ROOT/examples/wav2vec/unsupervised/scripts/merge_clusters.py $tgt_dir/precompute_pca$dim --cluster-dir $tgt_dir/CLUS128 \ - --split $split --save-dir $tgt_dir/precompute_pca${dim}_cls128_mean --pooling mean - - python $FAIRSEQ_ROOT/examples/wav2vec/unsupervised/scripts/mean_pool.py $tgt_dir/precompute_pca${dim}_cls128_mean \ - --save-dir $tgt_dir/precompute_pca${dim}_cls128_mean_pooled --split $split -done diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/prepare_text.sh b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/prepare_text.sh deleted file mode 100644 index 1caf13cb6a2a0bd84e5322c92124b2fa37368f9a..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/prepare_text.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env zsh -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -lg=$1 -text_path=$2 -target_dir=$3 -min_phones=$4 -phonemizer=$5 -lid_path=$6 - -if [ -z "$lid_path" ]; then - lid_path="lid.187.bin" -fi - -ph_lg=${lg:l} -if test "$lg" = 'fr'; then - ph_lg='fr-fr' -elif test "$lg" = 'en'; then - ph_lg='en-us' -elif test "$lg" = 'pt'; then - ph_lg='pt-br' -fi - -ESPEAK_PATH='' -if test "$phonemizer" = 'espeak'; then - ESPEAK_PATH=$(which espeak) -elif test "$phonemizer" = 'espeak-ng'; then - ESPEAK_PATH=$(which espeak-ng) -elif test "$phonemizer" = 'G2P'; then - ESPEAK_PATH='' -else - echo "Unknown phonemizer $phonemizer. Valid options are espeak, espean-ng and G2P" - exit 1 -fi - -echo $lg -echo $ph_lg -echo $text_path -echo $target_dir -echo "min phone seen threshold is $min_phones" - -mkdir -p $target_dir -python $FAIRSEQ_ROOT/examples/wav2vec/unsupervised/scripts/normalize_and_filter_text.py --lang $lg --fasttext-model $lid_path < $text_path | grep -v '\-\-\-' >! $target_dir/lm.upper.lid.txt -python $FAIRSEQ_ROOT/fairseq_cli/preprocess.py --dataset-impl mmap --trainpref $target_dir/lm.upper.lid.txt --only-source --destdir $target_dir --thresholdsrc 2 --padding-factor 1 --dict-only -cut -f1 -d' ' $target_dir/dict.txt | grep -v -x '[[:punct:]]*' | grep -Pv '\d\d\d\d\d+' >! $target_dir/words.txt - - -if [ -z "$ESPEAK_PATH" ]; then - python $FAIRSEQ_ROOT/examples/wav2vec/unsupervised/scripts/g2p_wrd_to_phn.py --compact < $target_dir/words.txt > $target_dir/phones.txt -else - # echoing 1 into corpus will prevent the mismatch lines between lexicon and phones in case the phonemizer fails - one=$(echo "1" | PHONEMIZER_ESPEAK_PATH=$ESPEAK_PATH phonemize -p ' ' -w '' -l $ph_lg --language-switch remove-flags) - sed 's/$/ 1/' $target_dir/words.txt | PHONEMIZER_ESPEAK_PATH=$ESPEAK_PATH phonemize -o $target_dir/phones.txt -p ' ' -w '' -l $ph_lg -j 70 --language-switch remove-flags - echo "one is ${one}" - sed -i "s/${one}$//" $target_dir/phones.txt -fi - -paste $target_dir/words.txt $target_dir/phones.txt >! $target_dir/lexicon.lst - -python $FAIRSEQ_ROOT/fairseq_cli/preprocess.py --dataset-impl mmap --trainpref $target_dir/phones.txt --only-source --destdir $target_dir/phones --thresholdsrc $min_phones --padding-factor 1 --dict-only - -python $FAIRSEQ_ROOT/examples/wav2vec/unsupervised/scripts/filter_lexicon.py -d $target_dir/phones/dict.txt < $target_dir/lexicon.lst >! $target_dir/lexicon_filtered.lst -python $FAIRSEQ_ROOT/examples/wav2vec/unsupervised/scripts/phonemize_with_sil.py -s 0.25 --surround --lexicon $target_dir/lexicon_filtered.lst < $target_dir/lm.upper.lid.txt >! $target_dir/phones/lm.phones.filtered.txt -cp $target_dir/phones/dict.txt $target_dir/phones/dict.phn.txt -echo "<SIL> 0" >> $target_dir/phones/dict.phn.txt -python $FAIRSEQ_ROOT/fairseq_cli/preprocess.py --dataset-impl mmap --trainpref $target_dir/phones/lm.phones.filtered.txt --workers 70 --only-source --destdir $target_dir/phones --srcdict $target_dir/phones/dict.phn.txt - -$KENLM_ROOT/lmplz -o 4 < $target_dir/lm.upper.lid.txt --discount_fallback --prune 0 0 0 3 >! $target_dir/kenlm.wrd.o40003.arpa -$KENLM_ROOT/build_binary $target_dir/kenlm.wrd.o40003.arpa $target_dir/kenlm.wrd.o40003.bin - -lg=$lg python $FAIRSEQ_ROOT/examples/speech_recognition/kaldi/kaldi_initializer.py kaldi_root=$KALDI_ROOT fst_dir=$target_dir/fst/phn_to_words_sil lm_arpa=$target_dir/kenlm.wrd.o40003.arpa wav2letter_lexicon=$target_dir/lexicon_filtered.lst data_dir=$target_dir/phones in_labels=phn "blank_symbol='<SIL>'" -lg=$lg python $FAIRSEQ_ROOT/examples/speech_recognition/kaldi/kaldi_initializer.py kaldi_root=$KALDI_ROOT fst_dir=$target_dir/fst/phn_to_words lm_arpa=$target_dir/kenlm.wrd.o40003.arpa wav2letter_lexicon=$target_dir/lexicon_filtered.lst data_dir=$target_dir/phones in_labels=phn - -$KENLM_ROOT/lmplz -o 4 < $target_dir/phones/lm.phones.filtered.txt --discount_fallback >! $target_dir/phones/lm.phones.filtered.04.arpa -$KENLM_ROOT/build_binary $target_dir/phones/lm.phones.filtered.04.arpa $target_dir/phones/lm.phones.filtered.04.bin -$KENLM_ROOT/lmplz -o 6 < $target_dir/phones/lm.phones.filtered.txt --discount_fallback >! $target_dir/phones/lm.phones.filtered.06.arpa -$KENLM_ROOT/build_binary $target_dir/phones/lm.phones.filtered.06.arpa $target_dir/phones/lm.phones.filtered.06.bin - -lg=$lg python $FAIRSEQ_ROOT/examples/speech_recognition/kaldi/kaldi_initializer.py kaldi_root=$KALDI_ROOT fst_dir=$target_dir/fst/phn_to_phn_sil lm_arpa=$target_dir/phones/lm.phones.filtered.06.arpa data_dir=$target_dir/phones in_labels=phn "blank_symbol='<SIL>'" diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/prepare_timit.sh b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/prepare_timit.sh deleted file mode 100644 index d8f5d596b4b4ec55f11a82dbbf83bad4a22c0b6c..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/prepare_timit.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/bin/bash -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -timit_root=$1 # assume it is the upper-cased version -tgt_dir=$2 -model=$3 - -set -eu - -setups="matched unmatched" -splits="test valid train train_text" - -tgt_dir=$(realpath $tgt_dir) -sph2wav=$KALDI_ROOT/tools/sph2pipe_v2.5/sph2pipe -wav_dir=$tgt_dir/wav - - -mkdir -p $tgt_dir $wav_dir -find $timit_root/{TRAIN,TEST} -iname "*.WAV" > $tgt_dir/all_sph.flist -cat $tgt_dir/all_sph.flist | sed -e 's#//*#/#g' -e 's#.*/\([^/]*\)/\([^/]*\).WAV#\1_\2#g' > $tgt_dir/all.uid -paste -d' ' $tgt_dir/{all_sph.flist,all.uid} | \ - awk -v sph2wav=$sph2wav -v wav_dir=$wav_dir '{print sph2wav " -f wav " $1 " > " wav_dir "/" $2 ".wav"}' \ - > $tgt_dir/sph2wav.sh -bash $tgt_dir/sph2wav.sh -cat $tgt_dir/all.uid | awk -v wav_dir=$(pwd)/$wav_dir '{print $1" "wav_dir"/"$1".wav"}' | sort > $tgt_dir/all_wav.scp -cut -d' ' -f2 $tgt_dir/all_wav.scp | xargs -I{} soxi -s {} > $tgt_dir/all.dur -paste -d' ' $tgt_dir/{all_wav.scp,all.dur} > $tgt_dir/all_wav_dur.scp -rm $tgt_dir/{all.uid,all_sph.flist,sph2wav.sh} - -find $timit_root/{TRAIN,TEST} -iname "*.PHN" > $tgt_dir/all_phn60.flist -while read line; do - if [ ! -f $line ]; then - >&2 echo "Cannot find transcription file '$line'" && exit 1; - fi - cut -f3 -d' ' "$line" | tr '\n' ' ' | perl -ape 's: *$:\n:;' -done < $tgt_dir/all_phn60.flist > $tgt_dir/all.phn60 -cat $tgt_dir/all_phn60.flist | sed -e 's#//*#/#g' -e 's#.*/\([^/]*\)/\([^/]*\).PHN#\1_\2#g' | \ - paste -d' ' - $tgt_dir/all.phn60 | \ - $KALDI_ROOT/egs/timit/s5/local/timit_norm_trans.pl -i - -m $KALDI_ROOT/egs/timit/s5/conf/phones.60-48-39.map -to 39 | \ - sort > $tgt_dir/all.phn -echo "done preparing wav and 39-phone transcripts" - - -for s in $setups; do - mkdir -p $tgt_dir/$s - for x in $splits; do - uid_path=config/timit_${s}/${x}.uid - grep -w -f $uid_path $tgt_dir/all.phn | cut -d' ' -f2- > $tgt_dir/$s/$x.phn - ln -sf $(realpath $tgt_dir/$s/$x.phn) $tgt_dir/$s/$x.wrd - - echo "/" > $tgt_dir/$s/$x.tsv && grep -w -f $uid_path $tgt_dir/all_wav_dur.scp | cut -d' ' -f2- | sed 's# #\t#' >> $tgt_dir/$s/$x.tsv - done - - for x in $splits; do - cat $tgt_dir/$s/$x.phn - done | tr ' ' '\n' | sort -u | awk '{print $1" "1}' > $tgt_dir/$s/dict.phn.txt - ln -sf $(realpath $tgt_dir/$s/dict.phn.txt) $tgt_dir/$s/dict.wrd.txt -done -echo "done preparing unmatched and matched setups for TIMIT" - - -for s in $setups; do - zsh scripts/prepare_audio.sh $tgt_dir/$s $tgt_dir/$s/feat $model - - lm_dir=$tgt_dir/$s/phones - fst_dir=$tgt_dir/$s/fst/phn_to_phn - - python $FAIRSEQ_ROOT/fairseq_cli/preprocess.py --dataset-impl mmap --trainpref $tgt_dir/$s/train_text.phn --workers 10 --only-source --destdir $lm_dir --srcdict $tgt_dir/$s/dict.phn.txt - $KENLM_ROOT/lmplz -o 3 < $tgt_dir/$s/train_text.phn --discount_fallback >$lm_dir/train_text_phn.03.arpa - $KENLM_ROOT/build_binary $lm_dir/train_text_phn.03.arpa $lm_dir/train_text_phn.03.bin - $KENLM_ROOT/lmplz -o 4 < $tgt_dir/$s/train_text.phn --discount_fallback >$lm_dir/train_text_phn.04.arpa - $KENLM_ROOT/build_binary $lm_dir/train_text_phn.04.arpa $lm_dir/train_text_phn.04.bin - - python $FAIRSEQ_ROOT/examples/speech_recognition/kaldi/kaldi_initializer.py kaldi_root=$KALDI_ROOT fst_dir=$fst_dir lm_arpa=$lm_dir/train_text_phn.03.arpa data_dir=$tgt_dir/$s in_labels=phn -done -echo "done preprocessing audio and text for wav2vec-U" diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/remove_silence.py b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/remove_silence.py deleted file mode 100644 index 5afdad7f5f4ee5f0b1e992eee1f81112c4fa9b34..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/remove_silence.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python3 -u -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -""" -get intervals from .vads file, specify output data, and this script removes silences and saves the audio data in out path folder -paths=shards/train.tsv -vads=shards/train.vads -python remove_silence.py --paths $paths --vads $vads -""" - -import argparse -import os - -import torch -import torchaudio -import tqdm - -parser = argparse.ArgumentParser() -parser.add_argument("--tsv", default="", type=str) -parser.add_argument("--vads", default="", type=str) -parser.add_argument("--out", type=str) -params = parser.parse_args() - -# load paths -paths = [] -with open(params.tsv) as f: - root = next(f).rstrip() - for line in f: - paths.append(os.path.join(root, line.rstrip().split("\t")[0])) - -# load vads -list_intervals = [] -with open(params.vads) as f: - for line in f: - interval = [ - [int(w.split(":")[0]), int(w.split(":")[1])] for w in line.rstrip().split() - ] - list_intervals.append(interval) - - -# load audio and keep only intervals (i.e. remove silences) -for i in tqdm.trange(len(paths)): - data, _ = torchaudio.load(paths[i]) - if len(list_intervals[i]) > 0: - data_filtered = torch.cat( - [data[0][int(it[0]) : int(it[1])] for it in list_intervals[i]] - ).unsqueeze(0) - else: - data_filtered = data - - # YOU MAY NEED TO MODIFY THIS TO GET THE RIGHT SUBPATH - # outpath = params.out + '/'.join(paths[i].split('/')[-1]) - outpath = params.out + "/" + "/".join(paths[i].split("/")[-2:]) - - if not os.path.isdir("/".join(outpath.split("/")[:-1])): - os.makedirs("/".join(outpath.split("/")[:-1])) - if not os.path.exists(outpath): - torchaudio.save(outpath, data_filtered, sample_rate=16000) - else: - print(outpath, "exists!") diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/vads.py b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/vads.py deleted file mode 100644 index 753c6928c9c1c31de86663bc3bca26d25e4f0df8..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/vads.py +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/env python3 -u -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import os.path as osp -import sys -from copy import deepcopy - -import numpy as np -import soundfile as sf -from scipy.signal import lfilter -from tqdm import tqdm - - -def get_parser(): - parser = argparse.ArgumentParser(description="compute vad segments") - parser.add_argument( - "--rvad-home", - "-r", - help="path to rvad home (see https://github.com/zhenghuatan/rVADfast)", - required=True, - ) - - return parser - - -def rvad(speechproc, path): - winlen, ovrlen, pre_coef, nfilter, nftt = 0.025, 0.01, 0.97, 20, 512 - ftThres = 0.5 - vadThres = 0.4 - opts = 1 - - data, fs = sf.read(path) - assert fs == 16_000, "sample rate must be 16khz" - ft, flen, fsh10, nfr10 = speechproc.sflux(data, fs, winlen, ovrlen, nftt) - - # --spectral flatness -- - pv01 = np.zeros(ft.shape[0]) - pv01[np.less_equal(ft, ftThres)] = 1 - pitch = deepcopy(ft) - - pvblk = speechproc.pitchblockdetect(pv01, pitch, nfr10, opts) - - # --filtering-- - ENERGYFLOOR = np.exp(-50) - b = np.array([0.9770, -0.9770]) - a = np.array([1.0000, -0.9540]) - fdata = lfilter(b, a, data, axis=0) - - # --pass 1-- - noise_samp, noise_seg, n_noise_samp = speechproc.snre_highenergy( - fdata, nfr10, flen, fsh10, ENERGYFLOOR, pv01, pvblk - ) - - # sets noisy segments to zero - for j in range(n_noise_samp): - fdata[range(int(noise_samp[j, 0]), int(noise_samp[j, 1]) + 1)] = 0 - - vad_seg = speechproc.snre_vad( - fdata, nfr10, flen, fsh10, ENERGYFLOOR, pv01, pvblk, vadThres - ) - return vad_seg, data - - -def main(): - parser = get_parser() - args = parser.parse_args() - - sys.path.append(args.rvad_home) - import speechproc - - stride = 160 - lines = sys.stdin.readlines() - root = lines[0].rstrip() - for fpath in tqdm(lines[1:]): - path = osp.join(root, fpath.split()[0]) - vads, wav = rvad(speechproc, path) - - start = None - vad_segs = [] - for i, v in enumerate(vads): - if start is None and v == 1: - start = i * stride - elif start is not None and v == 0: - vad_segs.append((start, i * stride)) - start = None - if start is not None: - vad_segs.append((start, len(wav))) - - print(" ".join(f"{v[0]}:{v[1]}" for v in vad_segs)) - - -if __name__ == "__main__": - main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/wav2vec_apply_cluster_faiss.py b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/wav2vec_apply_cluster_faiss.py deleted file mode 100644 index 0fac60cf0d548cb08584924b4921145cfce7fcf4..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/wav2vec_apply_cluster_faiss.py +++ /dev/null @@ -1,127 +0,0 @@ -#!/usr/bin/env python3 -u -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import os -import os.path as osp -import sys - -import faiss -import numpy as np -import torch -import torch.nn.functional as F -import tqdm -from wav2vec_cluster_faiss import Wav2VecFeatureReader, parse_faiss_specs - - -def get_parser(): - parser = argparse.ArgumentParser(description="apply clusters") - # fmt: off - parser.add_argument('data', help='location of tsv files') - parser.add_argument('--split', help='split to process', required=True) - parser.add_argument('--labels', help='split to process', default="phn") - parser.add_argument('--path', help='path to pca and centroids', required=True) - parser.add_argument('--checkpoint', type=str, help='checkpoint for wav2vec model (if using wav2vec features)', required=True) - parser.add_argument('--layer', '-l', type=int, help='which layer to read', default=14) - parser.add_argument('--max-tsz', type=int, help='batch kmeans up to this much', default=14) - # fmt: on - - return parser - - -def get_iterator(args): - label_path = osp.join(args.data, f"{args.split}.{args.labels}") - if osp.exists(label_path): - lp = open(label_path, "r") - else: - lp = None - - with open(osp.join(args.data, f"{args.split}.tsv"), "r") as fp: - lines = fp.read().split("\n") - root = lines.pop(0).strip() - files = [line.rstrip() for line in lines if len(line) > 0] - - if lp is not None: - lbls = [line.rstrip() for line in lp] - else: - lbls = [None] * len(files) - - num = len(files) - reader = Wav2VecFeatureReader(args.checkpoint, args.layer) - - def iterate(): - for fname, lbl in zip(files, lbls): - file = osp.join(root, fname.split("\t")[0]) - feats = reader.get_feats(file) - yield feats.data, fname, lbl - - return iterate, num, root - - -def main(): - parser = get_parser() - args = parser.parse_args() - - spec = osp.basename(args.path) - - try: - faiss_spec = parse_faiss_specs(spec.rstrip("/"))[0] - except: - print(spec) - raise - - print("Faiss Spec:", faiss_spec, file=sys.stderr) - - if faiss_spec.pca: - A = torch.from_numpy(np.load(osp.join(args.path, "pca_A.npy"))).cuda() - b = torch.from_numpy(np.load(osp.join(args.path, "pca_b.npy"))).cuda() - print("Loaded PCA", file=sys.stderr) - - centroids = np.load(osp.join(args.path, "centroids.npy")) - print("Loaded centroids", centroids.shape, file=sys.stderr) - - res = faiss.StandardGpuResources() - index_flat = ( - faiss.IndexFlatL2(centroids.shape[1]) - if not faiss_spec.sphere - else faiss.IndexFlatIP(centroids.shape[1]) - ) - faiss_index = faiss.index_cpu_to_gpu(res, 0, index_flat) - faiss_index.add(centroids) - - generator, num, root = get_iterator(args) - iterator = generator() - - had_labels = False - label_path = osp.join(args.path, f"{args.split}.{args.labels}") - - with torch.no_grad(): - with open(osp.join(args.path, f"{args.split}.src"), "w") as fp, open( - osp.join(args.path, f"{args.split}.tsv"), "w" - ) as pp, open(label_path, "w") as lp: - print(root, file=pp) - for f, fname, lbl in tqdm.tqdm(iterator, total=num): - if faiss_spec.pca: - f = torch.mm(f, A) + b - if faiss_spec.norm: - f = F.normalize(f, p=2, dim=-1) - - f = f.cpu().numpy() - - _, z = faiss_index.search(f, 1) - - print(" ".join(str(x.item()) for x in z), file=fp) - print(fname, file=pp) - - if lbl is not None: - print(lbl, file=lp) - had_labels = True - if not had_labels: - os.remove(label_path) - - -if __name__ == "__main__": - main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/wav2vec_cluster_faiss.py b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/wav2vec_cluster_faiss.py deleted file mode 100644 index d7adfe5dfc2827bb8ee80c626e3a9138cc1e6614..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/wav2vec_cluster_faiss.py +++ /dev/null @@ -1,208 +0,0 @@ -#!/usr/bin/env python3 -u -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import gc -import os -import os.path as osp -import random -from collections import namedtuple - -import fairseq -import faiss -import numpy as np -import soundfile as sf -import torch -import tqdm - - -def get_parser(): - parser = argparse.ArgumentParser( - description="compute kmeans codebook from kaldi-computed feats" - ) - # fmt: off - parser.add_argument('data', help='location of tsv files') - parser.add_argument('--save-dir', help='where to save the output', required=True) - parser.add_argument('--checkpoint', type=str, help='checkpoint for wav2vec model (if using wav2vec features)', required=True) - parser.add_argument('--sample-pct', '-r', type=float, help='percentage of timesteps to sample', default=0) - parser.add_argument('--layer', '-l', type=int, help='which layer to read', default=14) - parser.add_argument('--faiss-specs', '-f', type=str, - help='faiss index specs; separated by space ' - 'format is: PCAx_NORM_CLUSx_SPHERICAL -> ' - 'PCAx if exists first apply PCA ' - 'NORM if exists, normalize the vector by L2 norm ' - 'CLUSx must exist, cluster to x clusters ' - 'SPEHRICAL if exists, apply spherical kmeans', - default='l2') - # fmt: on - - return parser - - -faiss_spec = namedtuple("faiss_spec", ["pca", "norm", "n_clus", "sphere", "spec_str"]) - - -def parse_faiss_specs(specs_str): - specs = [] - for ss in specs_str.split(): - comps = ss.split("_") - pca = 0 - norm = False - n_clus = 0 - sphere = False - for c in comps: - if c.startswith("PCA"): - pca = int(c[3:]) - elif c == "NORM": - norm = True - elif c.startswith("CLUS"): - n_clus = int(c[4:]) - elif c == "SPHERICAL": - sphere = True - assert n_clus > 0 - specs.append( - faiss_spec(pca=pca, norm=norm, n_clus=n_clus, sphere=sphere, spec_str=ss) - ) - return specs - - -class Wav2VecFeatureReader(object): - def __init__(self, cp_file, layer): - state = fairseq.checkpoint_utils.load_checkpoint_to_cpu(cp_file) - - self.layer = layer - - if "cfg" in state: - w2v_args = state["cfg"] - task = fairseq.tasks.setup_task(w2v_args.task) - model = task.build_model(w2v_args.model) - else: - w2v_args = state["args"] - task = fairseq.tasks.setup_task(w2v_args) - model = task.build_model(w2v_args) - model.load_state_dict(state["model"], strict=True) - model.eval() - model.cuda() - self.model = model - - def read_audio(self, fname): - """Load an audio file and return PCM along with the sample rate""" - wav, sr = sf.read(fname) - assert sr == 16e3 - - return wav - - def get_feats(self, loc): - x = self.read_audio(loc) - with torch.no_grad(): - source = torch.from_numpy(x).view(1, -1).float().cuda() - res = self.model( - source=source, mask=False, features_only=True, layer=self.layer - ) - return res["layer_results"][self.layer][0].squeeze(1) - - -def get_iterator(args): - with open(args.data, "r") as fp: - lines = fp.read().split("\n") - root = lines.pop(0).strip() - files = [osp.join(root, line.split("\t")[0]) for line in lines if len(line) > 0] - - if getattr(args, "sample_pct", 0) > 0: - files = random.sample(files, int(args.sample_pct * len(files))) - num = len(files) - reader = Wav2VecFeatureReader(args.checkpoint, args.layer) - - def iterate(): - for fname in files: - feats = reader.get_feats(fname) - yield feats.cpu().numpy() - - return iterate, num - - -def main(): - parser = get_parser() - args = parser.parse_args() - - faiss_specs = parse_faiss_specs(args.faiss_specs) - print("Faiss Specs:", faiss_specs) - - feat_path = osp.join(args.save_dir, "features") - if osp.exists(feat_path + ".npy"): - feats = np.load(feat_path + ".npy") - else: - generator, num = get_iterator(args) - iterator = generator() - - feats = [] - for f in tqdm.tqdm(iterator, total=num): - feats.append(f) - - del iterator - del generator - - feats = np.concatenate(feats) - - print(feats.shape) - - os.makedirs(args.save_dir, exist_ok=True) - # np.save(feat_path, feats) - - gc.collect() - torch.cuda.empty_cache() - - reload = False - for spec in faiss_specs: - print("Processing spec", spec) - - if reload: - print("Reloading...") - del feats - gc.collect() - feats = np.load(feat_path + ".npy") - - save_path = osp.join(args.save_dir, spec.spec_str) - os.makedirs(save_path, exist_ok=True) - d = feats.shape[-1] - x = feats - if spec.pca > 0: - print("Computing PCA") - pca = faiss.PCAMatrix(d, spec.pca) - pca.train(x) - d = spec.pca - b = faiss.vector_to_array(pca.b) - A = faiss.vector_to_array(pca.A).reshape(pca.d_out, pca.d_in) - np.save(osp.join(save_path, "pca_A"), A.T) - np.save(osp.join(save_path, "pca_b"), b) - print("Applying PCA") - x = pca.apply_py(x) - - if spec.norm: - reload = spec.pca <= 0 - print("Normalizing") - faiss.normalize_L2(x) - - print("Computing kmeans") - kmeans = faiss.Kmeans( - d, - spec.n_clus, - niter=50, - verbose=True, - spherical=spec.sphere, - max_points_per_centroid=feats.shape[0], - gpu=True, - nredo=3, - ) - kmeans.train(x) - np.save(osp.join(save_path, "centroids"), kmeans.centroids) - del kmeans - del x - gc.collect() - - -if __name__ == "__main__": - main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/wav2vec_extract_features.py b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/wav2vec_extract_features.py deleted file mode 100644 index 6481025b876a5daf6978e5741dadc4c6e97ae3aa..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/wav2vec_extract_features.py +++ /dev/null @@ -1,118 +0,0 @@ -#!/usr/bin/env python3 -u -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import os -import os.path as osp -from shutil import copyfile - -import fairseq -import soundfile as sf -import torch -import torch.nn.functional as F -import tqdm -from npy_append_array import NpyAppendArray - - -def get_parser(): - parser = argparse.ArgumentParser( - description="compute kmeans codebook from kaldi-computed feats" - ) - # fmt: off - parser.add_argument('data', help='location of tsv files') - parser.add_argument('--split', help='which split to read', required=True) - parser.add_argument('--save-dir', help='where to save the output', required=True) - parser.add_argument('--checkpoint', type=str, help='checkpoint for wav2vec ctc model', required=True) - parser.add_argument('--layer', type=int, default=14, help='which layer to use') - # fmt: on - - return parser - - -class Wav2VecFeatureReader(object): - def __init__(self, cp_file, layer): - model, cfg, task = fairseq.checkpoint_utils.load_model_ensemble_and_task( - [cp_file] - ) - model = model[0] - model.eval() - model.cuda() - self.model = model - self.task = task - self.layer = layer - - def read_audio(self, fname): - """Load an audio file and return PCM along with the sample rate""" - wav, sr = sf.read(fname) - assert sr == 16e3 - - return wav - - def get_feats(self, loc): - x = self.read_audio(loc) - with torch.no_grad(): - source = torch.from_numpy(x).float().cuda() - if self.task.cfg.normalize: - assert source.dim() == 1, source.dim() - with torch.no_grad(): - source = F.layer_norm(source, source.shape) - source = source.view(1, -1) - - m_res = self.model(source=source, mask=False, features_only=True, layer=self.layer) - return m_res["x"].squeeze(0).cpu() - - -def get_iterator(args): - with open(osp.join(args.data, args.split) + ".tsv", "r") as fp: - lines = fp.read().split("\n") - root = lines.pop(0).strip() - files = [osp.join(root, line.split("\t")[0]) for line in lines if len(line) > 0] - - num = len(files) - reader = Wav2VecFeatureReader(args.checkpoint, args.layer) - - def iterate(): - for fname in files: - w2v_feats = reader.get_feats(fname) - yield w2v_feats - - return iterate, num - - -def main(): - parser = get_parser() - args = parser.parse_args() - - os.makedirs(args.save_dir, exist_ok=True) - - def create_files(dest): - copyfile(osp.join(args.data, args.split) + ".tsv", dest + ".tsv") - if osp.exists(osp.join(args.data, args.split) + ".wrd"): - copyfile(osp.join(args.data, args.split) + ".wrd", dest + ".wrd") - if osp.exists(osp.join(args.data, args.split) + ".phn"): - copyfile(osp.join(args.data, args.split) + ".phn", dest + ".phn") - - if osp.exists(dest + ".npy"): - os.remove(dest + ".npy") - npaa = NpyAppendArray(dest + ".npy") - return npaa - - save_path = osp.join(args.save_dir, args.split) - npaa = create_files(save_path) - - generator, num = get_iterator(args) - iterator = generator() - - with open(save_path + ".lengths", "w") as l_f: - for w2v_feats in tqdm.tqdm(iterator, total=num): - print(len(w2v_feats), file=l_f) - - if len(w2v_feats) > 0: - npaa.append(w2v_feats.numpy()) - - -if __name__ == "__main__": - main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/wer.py b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/wer.py deleted file mode 100644 index 613ab50d39019f6edf67c56c2353646be2a2f17d..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/wer.py +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env python3 -u -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -""" -Implement unsupervised metric for decoding hyperparameter selection: - $$ alpha * LM_PPL + ViterbitUER(%) * 100 $$ -""" -import argparse -import logging -import sys - -import editdistance - -logging.root.setLevel(logging.INFO) -logging.basicConfig(stream=sys.stdout, level=logging.INFO) -logger = logging.getLogger(__name__) - - -def get_parser(): - parser = argparse.ArgumentParser() - parser.add_argument("-s", "--hypo", help="hypo transcription", required=True) - parser.add_argument( - "-r", "--reference", help="reference transcription", required=True - ) - return parser - - -def compute_wer(ref_uid_to_tra, hyp_uid_to_tra, g2p): - d_cnt = 0 - w_cnt = 0 - w_cnt_h = 0 - for uid in hyp_uid_to_tra: - ref = ref_uid_to_tra[uid].split() - if g2p is not None: - hyp = g2p(hyp_uid_to_tra[uid]) - hyp = [p for p in hyp if p != "'" and p != " "] - hyp = [p[:-1] if p[-1].isnumeric() else p for p in hyp] - else: - hyp = hyp_uid_to_tra[uid].split() - d_cnt += editdistance.eval(ref, hyp) - w_cnt += len(ref) - w_cnt_h += len(hyp) - wer = float(d_cnt) / w_cnt - logger.debug( - ( - f"wer = {wer * 100:.2f}%; num. of ref words = {w_cnt}; " - f"num. of hyp words = {w_cnt_h}; num. of sentences = {len(ref_uid_to_tra)}" - ) - ) - return wer - - -def main(): - args = get_parser().parse_args() - - errs = 0 - count = 0 - with open(args.hypo, "r") as hf, open(args.reference, "r") as rf: - for h, r in zip(hf, rf): - h = h.rstrip().split() - r = r.rstrip().split() - errs += editdistance.eval(r, h) - count += len(r) - - logger.info(f"UER: {errs / count * 100:.2f}%") - - -if __name__ == "__main__": - main() - - -def load_tra(tra_path): - with open(tra_path, "r") as f: - uid_to_tra = {} - for line in f: - uid, tra = line.split(None, 1) - uid_to_tra[uid] = tra - logger.debug(f"loaded {len(uid_to_tra)} utterances from {tra_path}") - return uid_to_tra diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/wrd_to_ltr.py b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/wrd_to_ltr.py deleted file mode 100644 index f83471409a434556cab70086ca9e2d72d4bdddd5..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/scripts/wrd_to_ltr.py +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env python3 -u -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import sys - - -def main(): - for line in sys.stdin: - print(" ".join(list(line.strip().replace(" ", "|"))) + " |") - - -if __name__ == "__main__": - main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/tasks/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/tasks/__init__.py deleted file mode 100644 index 237841ac0f6827fc095fa8d94944a2b54aa5ae69..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/tasks/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from .unpaired_audio_text import UnpairedAudioText - -__all__ = [ - "UnpairedAudioText", -] diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/tasks/unpaired_audio_text.py b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/tasks/unpaired_audio_text.py deleted file mode 100644 index 14635db821741858fd0896942ed621c48a4ec778..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/tasks/unpaired_audio_text.py +++ /dev/null @@ -1,445 +0,0 @@ -# Copyright (c) 2017-present, Facebook, Inc. -# All rights reserved. -# -# This source code is licensed under the license found in the LICENSE file in -# the root directory of this source tree. An additional grant of patent rights -# can be found in the PATENTS file in the same directory. - -import logging -import math -import os -from dataclasses import dataclass, field -from typing import Optional - -import torch -from examples.speech_recognition.kaldi.kaldi_decoder import ( - KaldiDecoder, - KaldiDecoderConfig, -) -from fairseq.data import ( - Dictionary, - StripTokenDataset, - data_utils, -) -from fairseq.dataclass import FairseqDataclass -from fairseq.distributed.utils import get_data_parallel_world_size -from fairseq.logging import metrics -from fairseq.tasks import FairseqTask, register_task -from omegaconf import MISSING - -from ..data import ExtractedFeaturesDataset, RandomInputDataset - -logger = logging.getLogger(__name__) - - -@dataclass -class DecodingConfig(FairseqDataclass): - kenlm_path: Optional[str] = None - lm_weight: float = 0 - blank_weight: float = 0 - - -@dataclass -class UnpairedAudioTextConfig(FairseqDataclass): - data: str = field( - default=MISSING, metadata={"help": "path to data directory containing audio"} - ) - text_data: str = field( - default=MISSING, metadata={"help": "path to data directory containing text"} - ) - max_length: Optional[int] = None - labels: Optional[str] = field( - default=None, - metadata={"help": "extension of the label file to load, used for fine-tuning"}, - ) - unfiltered: bool = field( - default=False, metadata={"help": "load data with _unfiltered suffix"} - ) - ctc_eval: bool = field( - default=False, metadata={"help": "eval UER as if computed by CTC"} - ) - sort_by_length: bool = field( - default=True, metadata={"help": "sort examples by length of audio timesteps"} - ) - shuffle: bool = field(default=True, metadata={"help": "shuffle examples"}) - append_eos: bool = field(default=False, metadata={"help": "append eos"}) - uppercase: Optional[bool] = field( - default=False, metadata={"help": "uppercase for LM score computation"} - ) - skipwords: Optional[str] = field( - default="", - metadata={ - "help": "comma-separated words to be removed for LM score computation" - }, - ) - kenlm_path: Optional[str] = None - vocab_usage_power: float = 2 - - word_decoder_config: Optional[KaldiDecoderConfig] = None - word_kenlm_path: Optional[str] = None - - decoding_config: DecodingConfig = DecodingConfig() - - -@register_task("unpaired_audio_text", dataclass=UnpairedAudioTextConfig) -class UnpairedAudioText(FairseqTask): - """ """ - - cfg: UnpairedAudioTextConfig - - def __init__( - self, - cfg: UnpairedAudioTextConfig, - source_dictionary=None, - target_dictionary=None, - ): - super().__init__(cfg) - - self._target_dictionary = target_dictionary - self._source_dictionary = source_dictionary - self.num_symbols = ( - len([s for s in target_dictionary.symbols if not s.startswith("madeup")]) - - target_dictionary.nspecial - ) - self.sil_id = ( - target_dictionary.index("<SIL>") if "<SIL>" in target_dictionary else -1 - ) - self.kenlm = None - if cfg.kenlm_path is not None: - import kenlm - - self.kenlm = kenlm.Model(cfg.kenlm_path) - - self.word_kenlm = None - if cfg.word_kenlm_path is not None: - import kenlm - - self.word_kenlm = kenlm.Model(cfg.word_kenlm_path) - - self.uppercase = cfg.uppercase - self.skipwords = set(cfg.skipwords.split(",")) - - def str_postprocess(s): - s = " ".join(w for w in s.split() if w not in self.skipwords) - s = s.upper() if self.uppercase else s - return s - - self.str_postprocess = str_postprocess - self.compute_lm_score = lambda s: self.kenlm.score(self.str_postprocess(s)) - - self.compute_word_score = None - if cfg.word_decoder_config is not None: - self.kaldi_decoder = KaldiDecoder(cfg.word_decoder_config, beam=10) - - def compute_word_score(logits, padding): - res = self.kaldi_decoder.decode(logits, padding) - for r in res: - r = r.result() - assert len(r) == 1 - r = r[0] - yield r["score"], r["words"] - - self.compute_word_score = compute_word_score - - @classmethod - def setup_task(cls, cfg: UnpairedAudioTextConfig, **kwargs): - """Setup the task (e.g., load dictionaries). - - Args: - cfg (AudioPretrainingConfig): configuration of this task - """ - - dict_path = os.path.join(cfg.text_data, "dict.txt") - if os.path.exists(dict_path): - target_dictionary = Dictionary.load(dict_path) - else: - dict_path = os.path.join(cfg.data, f"dict.{cfg.labels}.txt") - target_dictionary = Dictionary.load(dict_path) - - return cls(cfg, target_dictionary=target_dictionary) - - def optimizer_step(self, optimizer, model, update_num): - if hasattr(model, "get_groups_for_update"): - groups = model.get_groups_for_update(update_num) - optimizer.step(groups={groups}) - else: - optimizer.step() - - def valid_step(self, sample, model, criterion): - res = model( - **sample["net_input"], - dense_x_only=True, - ) - - dense_x = res["logits"] - padding_mask = res["padding_mask"] - - word_scores = None - if self.compute_word_score is not None: - word_scores = self.compute_word_score(dense_x.cpu(), padding_mask.cpu()) - - z = dense_x.argmax(-1) - z[padding_mask] = self.target_dictionary.pad() - - vocab_seen = torch.zeros(self.num_symbols, dtype=torch.bool) - - import editdistance - - c_err = 0 - c_len = 0 - pred_c_len = 0 - lm_score_sum = 0 - for i, (x, t, id) in enumerate( - zip( - z, - sample["target"] if "target" in sample else [None] * len(z), - sample["id"], - ) - ): - - if t is not None: - t = t[(t >= self.target_dictionary.nspecial)] - x = x[ - (x >= self.target_dictionary.nspecial) - & (x < (self.num_symbols + self.target_dictionary.nspecial)) - ] - if self.sil_id >= 0: - x = x[x != self.sil_id] - - vocab_seen[x - self.target_dictionary.nspecial] = True - - pred_units_arr = x - if self.cfg.ctc_eval: - pred_units_arr = pred_units_arr.unique_consecutive() - pred_units_arr = pred_units_arr[pred_units_arr != 0] - - if id == 0: - if t is not None: - logger.info(f"REF: {self.target_dictionary.string(t)}") - logger.info(f"HYP: {self.target_dictionary.string(pred_units_arr)}") - - if self.kenlm is not None: - if t is not None: - ref_lm_s = self.compute_lm_score( - self.target_dictionary.string(t) - ) - logger.info( - f"LM [REF]: {ref_lm_s}, {math.pow(10, -ref_lm_s / (len(t) + 1))}" - ) - - hyp_lm_s = self.compute_lm_score( - self.target_dictionary.string(pred_units_arr) - ) - logger.info( - f"LM [HYP]: {hyp_lm_s}, {math.pow(10, -hyp_lm_s / (len(pred_units_arr) + 1))}" - ) - - pred_units_arr = pred_units_arr.tolist() - - pred_c_len += len(pred_units_arr) - - if t is not None: - t = t.tolist() - c_err += editdistance.eval(pred_units_arr, t) - c_len += len(t) - else: - c_len = pred_c_len - - if self.kenlm is not None: - pred_str = self.target_dictionary.string(pred_units_arr) - lm_score = self.compute_lm_score(pred_str) - lm_score_sum += lm_score - - kaldi_score_sum = 0 - word_lm_sum = 0 - num_words = 0 - if word_scores is not None: - for score, words in word_scores: - kaldi_score_sum += score - num_words += len(words) - if self.word_kenlm is not None: - word_lm_sum += self.kenlm.score(" ".join(words)) - - try: - world_size = get_data_parallel_world_size() - except: - world_size = 1 - - logging_output = { - "loss": c_err, - "_num_char_errors": c_err, - "_num_chars": c_len, - "_num_pred_chars": pred_c_len, - "ntokens": c_len, - "nsentences": z.size(0), - "sample_size": c_len, - "_world_size": world_size, - "_lm_score_sum": lm_score_sum, - "_kaldi_score_sum": kaldi_score_sum, - "_word_lm_sum": word_lm_sum, - "_num_words": num_words, - "_vocab_seen": vocab_seen, - } - - return c_err, c_len, logging_output - - def load_dataset(self, split: str, task_cfg: FairseqDataclass = None, **kwargs): - data_path = self.cfg.data - task_cfg = task_cfg or self.cfg - - has_unpaired_text = os.path.exists( - os.path.join(self.cfg.text_data, f"{split}.idx") - ) - - self.datasets[split] = ExtractedFeaturesDataset( - path=data_path, - split=split, - min_length=3, - max_length=task_cfg.max_length, - labels=None if has_unpaired_text else task_cfg.labels, - label_dict=self.target_dictionary, - shuffle=getattr(task_cfg, "shuffle", True), - sort_by_length=task_cfg.sort_by_length, - ) - - logger.info(f"split {split} has unpaired text? {has_unpaired_text}") - if has_unpaired_text: - text_dataset = data_utils.load_indexed_dataset( - os.path.join(self.cfg.text_data, split), self.target_dictionary - ) - text_dataset = StripTokenDataset(text_dataset, self.target_dictionary.eos()) - self.datasets[split] = RandomInputDataset( - self.datasets[split], - text_dataset, - ["random_label"], - add_to_input=True, - pad_idx=self.target_dictionary.pad(), - ) - - @property - def source_dictionary(self): - return self._source_dictionary - - @property - def target_dictionary(self): - """Return the :class:`~fairseq.data.Dictionary` for the language - model.""" - return self._target_dictionary - - def max_positions(self): - """Maximum input length supported by the encoder.""" - return None - - def reduce_metrics(self, logging_outputs, criterion): - super().reduce_metrics(logging_outputs, criterion) - - zero = torch.scalar_tensor(0.0) - num_char_errors = sum( - log.get("_num_char_errors", zero) for log in logging_outputs - ) - num_chars = sum(log.get("_num_chars", zero) for log in logging_outputs) - num_word_errors = sum( - log.get("_num_word_errors", zero) for log in logging_outputs - ) - num_words = sum(log.get("_num_words", zero) for log in logging_outputs) - num_pred_chars = sum( - log.get("_num_pred_chars", zero) for log in logging_outputs - ) - - lm_score_sum = sum(log.get("_lm_score_sum", zero) for log in logging_outputs) - vocab_seen = ( - sum(log.get("_vocab_seen", zero) for log in logging_outputs) - .bool() - .sum() - .item() - ) - kaldi_score_sum = sum( - log.get("_kaldi_score_sum", zero) for log in logging_outputs - ) - word_lm_sum = sum(log.get("_word_lm_sum", zero) for log in logging_outputs) - - metrics.log_scalar_sum("_num_char_errors", num_char_errors) - metrics.log_scalar_sum("_num_chars", num_chars) - metrics.log_scalar_sum("_num_word_errors", num_word_errors) - metrics.log_scalar_sum("_num_words", num_words) - - metrics.log_scalar_sum("lm_score_sum", lm_score_sum) - metrics.log_scalar_sum("num_pred_chars", num_pred_chars) - - if self.cfg.word_kenlm_path is not None: - metrics.log_scalar_sum("kaldi_score_sum", kaldi_score_sum) - metrics.log_scalar_sum("word_lm_sum", word_lm_sum) - - if num_chars > 0: - metrics.log_derived( - "uer", - lambda meters: meters["_num_char_errors"].sum - * 100.0 - / meters["_num_chars"].sum - if meters["_num_chars"].sum > 0 - else float("nan"), - ) - - if lm_score_sum < 0 and vocab_seen > 0: - metrics.log_scalar("vocab_seen_pct", vocab_seen / self.num_symbols) - - metrics.log_derived( - "weighted_lm_ppl", - lambda meters: math.pow( - 10, - -meters["lm_score_sum"].sum - / ( - meters["num_pred_chars"].sum + meters["nsentences"].sum - ), # account for </s> - ) - / meters["vocab_seen_pct"].avg ** self.cfg.vocab_usage_power, - ) - - metrics.log_derived( - "lm_ppl", - lambda meters: math.pow( - 10, - -meters["lm_score_sum"].sum - / ( - meters["num_pred_chars"].sum + meters["nsentences"].sum - ), # account for </s> - ), - ) - else: - metrics.log_derived("weighted_lm_ppl", lambda meters: float("inf")) - - if num_words > 0: - if word_lm_sum != 0: - metrics.log_derived( - "word_lm_ppl", - lambda meters: math.pow( - 10, - -meters["word_lm_sum"].sum - / ( - meters["_num_words"].sum + meters["nsentences"].sum - ), # account for </s> - ), - ) - metrics.log_derived( - "weighted_word_lm_ppl", - lambda meters: math.pow( - 10, - -meters["word_lm_sum"].sum - / ( - meters["_num_words"].sum + meters["nsentences"].sum - ), # account for </s> - ) - / meters["vocab_seen_pct"].avg ** self.cfg.vocab_usage_power, - ) - - if self.cfg.word_kenlm_path is not None: - metrics.log_derived( - "kaldi_score", - lambda meters: meters["kaldi_score_sum"].sum - / meters["nsentences"].sum, - ) - - def build_model(self, cfg: FairseqDataclass, from_checkpoint=False): - model = super().build_model(cfg) - - return model diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/w2vu_generate.py b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/w2vu_generate.py deleted file mode 100644 index 2fa4f6a0ce0d3520767bc5b5058cf3252a6608ba..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/unsupervised/w2vu_generate.py +++ /dev/null @@ -1,703 +0,0 @@ -#!/usr/bin/env python3 -u -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -""" -Run inference for pre-processed data with a trained model. -""" - -import ast -import logging -import math -import os -import sys -from collections import namedtuple -from dataclasses import dataclass, field -from enum import Enum, auto -from typing import Optional - -import editdistance -import hydra -import torch -from examples.speech_recognition.kaldi.kaldi_decoder import KaldiDecoderConfig -from fairseq import checkpoint_utils, progress_bar, tasks, utils -from fairseq.data.data_utils import post_process -from fairseq.dataclass.configs import FairseqConfig, FairseqDataclass -from fairseq.logging.meters import StopwatchMeter -from hydra.core.config_store import ConfigStore -from hydra.core.hydra_config import HydraConfig -from omegaconf import OmegaConf, open_dict - -logging.root.setLevel(logging.INFO) -logging.basicConfig(stream=sys.stdout, level=logging.INFO) -logger = logging.getLogger(__name__) - - -class DecoderType(Enum): - VITERBI = auto() - KENLM = auto() - FAIRSEQ = auto() - KALDI = auto() - - -@dataclass -class UnsupGenerateConfig(FairseqDataclass): - fairseq: FairseqConfig = FairseqConfig() - lm_weight: float = field( - default=2.0, - metadata={"help": "language model weight"}, - ) - w2l_decoder: DecoderType = field( - default=DecoderType.VITERBI, - metadata={"help": "type of decoder to use"}, - ) - kaldi_decoder_config: Optional[KaldiDecoderConfig] = None - lexicon: Optional[str] = field( - default=None, - metadata={ - "help": "path to lexicon. This is also used to 'phonemize' for unsupvised param tuning" - }, - ) - lm_model: Optional[str] = field( - default=None, - metadata={"help": "path to language model (kenlm or fairseq)"}, - ) - unit_lm: bool = field( - default=False, - metadata={"help": "whether to use unit lm"}, - ) - beam_threshold: float = field( - default=50.0, - metadata={"help": "beam score threshold"}, - ) - beam_size_token: float = field( - default=100.0, - metadata={"help": "max tokens per beam"}, - ) - beam: int = field( - default=5, - metadata={"help": "decoder beam size"}, - ) - nbest: int = field( - default=1, - metadata={"help": "number of results to return"}, - ) - word_score: float = field( - default=1.0, - metadata={"help": "word score to add at end of word"}, - ) - unk_weight: float = field( - default=-math.inf, - metadata={"help": "unknown token weight"}, - ) - sil_weight: float = field( - default=0.0, - metadata={"help": "silence token weight"}, - ) - targets: Optional[str] = field( - default=None, - metadata={"help": "extension of ground truth labels to compute UER"}, - ) - results_path: Optional[str] = field( - default=None, - metadata={"help": "where to store results"}, - ) - post_process: Optional[str] = field( - default=None, - metadata={"help": "how to post process results"}, - ) - vocab_usage_power: float = field( - default=2, - metadata={"help": "for unsupervised param tuning"}, - ) - - viterbi_transcript: Optional[str] = field( - default=None, - metadata={"help": "for unsupervised param tuning"}, - ) - min_lm_ppl: float = field( - default=0, - metadata={"help": "for unsupervised param tuning"}, - ) - min_vt_uer: float = field( - default=0, - metadata={"help": "for unsupervised param tuning"}, - ) - - blank_weight: float = field( - default=0, - metadata={"help": "value to add or set for blank emission"}, - ) - blank_mode: str = field( - default="set", - metadata={ - "help": "can be add or set, how to modify blank emission with blank weight" - }, - ) - sil_is_blank: bool = field( - default=False, - metadata={"help": "if true, <SIL> token is same as blank token"}, - ) - - unsupervised_tuning: bool = field( - default=False, - metadata={ - "help": "if true, returns a score based on unsupervised param selection metric instead of UER" - }, - ) - is_ax: bool = field( - default=False, - metadata={ - "help": "if true, assumes we are using ax for tuning and returns a tuple for ax to consume" - }, - ) - - -def get_dataset_itr(cfg, task): - return task.get_batch_iterator( - dataset=task.dataset(cfg.fairseq.dataset.gen_subset), - max_tokens=cfg.fairseq.dataset.max_tokens, - max_sentences=cfg.fairseq.dataset.batch_size, - max_positions=(sys.maxsize, sys.maxsize), - ignore_invalid_inputs=cfg.fairseq.dataset.skip_invalid_size_inputs_valid_test, - required_batch_size_multiple=cfg.fairseq.dataset.required_batch_size_multiple, - num_shards=cfg.fairseq.dataset.num_shards, - shard_id=cfg.fairseq.dataset.shard_id, - num_workers=cfg.fairseq.dataset.num_workers, - data_buffer_size=cfg.fairseq.dataset.data_buffer_size, - ).next_epoch_itr(shuffle=False) - - -def process_predictions( - cfg: UnsupGenerateConfig, - hypos, - tgt_dict, - target_tokens, - res_files, -): - retval = [] - word_preds = [] - transcriptions = [] - dec_scores = [] - - for i, hypo in enumerate(hypos[: min(len(hypos), cfg.nbest)]): - if torch.is_tensor(hypo["tokens"]): - tokens = hypo["tokens"].int().cpu() - tokens = tokens[tokens >= tgt_dict.nspecial] - hyp_pieces = tgt_dict.string(tokens) - else: - hyp_pieces = " ".join(hypo["tokens"]) - - if "words" in hypo and len(hypo["words"]) > 0: - hyp_words = " ".join(hypo["words"]) - else: - hyp_words = post_process(hyp_pieces, cfg.post_process) - - to_write = {} - if res_files is not None: - to_write[res_files["hypo.units"]] = hyp_pieces - to_write[res_files["hypo.words"]] = hyp_words - - tgt_words = "" - if target_tokens is not None: - if isinstance(target_tokens, str): - tgt_pieces = tgt_words = target_tokens - else: - tgt_pieces = tgt_dict.string(target_tokens) - tgt_words = post_process(tgt_pieces, cfg.post_process) - - if res_files is not None: - to_write[res_files["ref.units"]] = tgt_pieces - to_write[res_files["ref.words"]] = tgt_words - - if not cfg.fairseq.common_eval.quiet: - logger.info(f"HYPO {i}:" + hyp_words) - if tgt_words: - logger.info("TARGET:" + tgt_words) - - if "am_score" in hypo and "lm_score" in hypo: - logger.info( - f"DECODER AM SCORE: {hypo['am_score']}, DECODER LM SCORE: {hypo['lm_score']}, DECODER SCORE: {hypo['score']}" - ) - elif "score" in hypo: - logger.info(f"DECODER SCORE: {hypo['score']}") - - logger.info("___________________") - - hyp_words_arr = hyp_words.split() - tgt_words_arr = tgt_words.split() - - retval.append( - ( - editdistance.eval(hyp_words_arr, tgt_words_arr), - len(hyp_words_arr), - len(tgt_words_arr), - hyp_pieces, - hyp_words, - ) - ) - word_preds.append(hyp_words_arr) - transcriptions.append(to_write) - dec_scores.append(-hypo.get("score", 0)) # negate cuz kaldi returns NLL - - if len(retval) > 1: - best = None - for r, t in zip(retval, transcriptions): - if best is None or r[0] < best[0][0]: - best = r, t - for dest, tran in best[1].items(): - print(tran, file=dest) - dest.flush() - return best[0] - - assert len(transcriptions) == 1 - for dest, tran in transcriptions[0].items(): - print(tran, file=dest) - - return retval[0] - - -def prepare_result_files(cfg: UnsupGenerateConfig): - def get_res_file(file_prefix): - if cfg.fairseq.dataset.num_shards > 1: - file_prefix = f"{cfg.fairseq.dataset.shard_id}_{file_prefix}" - path = os.path.join( - cfg.results_path, - "{}{}.txt".format( - cfg.fairseq.dataset.gen_subset, - file_prefix, - ), - ) - return open(path, "w", buffering=1) - - if not cfg.results_path: - return None - - return { - "hypo.words": get_res_file(""), - "hypo.units": get_res_file("_units"), - "ref.words": get_res_file("_ref"), - "ref.units": get_res_file("_ref_units"), - "hypo.nbest.words": get_res_file("_nbest_words"), - } - - -def optimize_models(cfg: UnsupGenerateConfig, use_cuda, models): - """Optimize ensemble for generation""" - for model in models: - model.eval() - if cfg.fairseq.common.fp16: - model.half() - if use_cuda: - model.cuda() - - -GenResult = namedtuple( - "GenResult", - [ - "count", - "errs_t", - "gen_timer", - "lengths_hyp_unit_t", - "lengths_hyp_t", - "lengths_t", - "lm_score_t", - "num_feats", - "num_sentences", - "num_symbols", - "vt_err_t", - "vt_length_t", - ], -) - - -def generate(cfg: UnsupGenerateConfig, models, saved_cfg, use_cuda): - task = tasks.setup_task(cfg.fairseq.task) - saved_cfg.task.labels = cfg.fairseq.task.labels - task.load_dataset(cfg.fairseq.dataset.gen_subset, task_cfg=saved_cfg.task) - # Set dictionary - tgt_dict = task.target_dictionary - logger.info( - "| {} {} {} examples".format( - cfg.fairseq.task.data, - cfg.fairseq.dataset.gen_subset, - len(task.dataset(cfg.fairseq.dataset.gen_subset)), - ) - ) - # Load dataset (possibly sharded) - itr = get_dataset_itr(cfg, task) - # Initialize generator - gen_timer = StopwatchMeter() - - def build_generator(cfg: UnsupGenerateConfig): - w2l_decoder = cfg.w2l_decoder - if w2l_decoder == DecoderType.VITERBI: - from examples.speech_recognition.w2l_decoder import W2lViterbiDecoder - - return W2lViterbiDecoder(cfg, task.target_dictionary) - elif w2l_decoder == DecoderType.KENLM: - from examples.speech_recognition.w2l_decoder import W2lKenLMDecoder - - return W2lKenLMDecoder(cfg, task.target_dictionary) - elif w2l_decoder == DecoderType.FAIRSEQ: - from examples.speech_recognition.w2l_decoder import W2lFairseqLMDecoder - - return W2lFairseqLMDecoder(cfg, task.target_dictionary) - elif w2l_decoder == DecoderType.KALDI: - from examples.speech_recognition.kaldi.kaldi_decoder import KaldiDecoder - - assert cfg.kaldi_decoder_config is not None - - return KaldiDecoder( - cfg.kaldi_decoder_config, - cfg.beam, - ) - else: - raise NotImplementedError( - "only wav2letter decoders with (viterbi, kenlm, fairseqlm) options are supported at the moment but found " - + str(w2l_decoder) - ) - - generator = build_generator(cfg) - - kenlm = None - fairseq_lm = None - if cfg.lm_model is not None: - import kenlm - - kenlm = kenlm.Model(cfg.lm_model) - - num_sentences = 0 - if cfg.results_path is not None and not os.path.exists(cfg.results_path): - os.makedirs(cfg.results_path) - - res_files = prepare_result_files(cfg) - errs_t = 0 - lengths_hyp_t = 0 - lengths_hyp_unit_t = 0 - lengths_t = 0 - count = 0 - num_feats = 0 - all_hyp_pieces = [] - all_hyp_words = [] - - num_symbols = ( - len([s for s in tgt_dict.symbols if not s.startswith("madeup")]) - - tgt_dict.nspecial - ) - targets = None - if cfg.targets is not None: - tgt_path = os.path.join( - cfg.fairseq.task.data, cfg.fairseq.dataset.gen_subset + "." + cfg.targets - ) - if os.path.exists(tgt_path): - with open(tgt_path, "r") as f: - targets = f.read().splitlines() - viterbi_transcript = None - if cfg.viterbi_transcript is not None and len(cfg.viterbi_transcript) > 0: - logger.info(f"loading viterbi transcript from {cfg.viterbi_transcript}") - with open(cfg.viterbi_transcript, "r") as vf: - viterbi_transcript = vf.readlines() - viterbi_transcript = [v.rstrip().split() for v in viterbi_transcript] - - gen_timer.start() - - start = 0 - end = len(itr) - - hypo_futures = None - if cfg.w2l_decoder == DecoderType.KALDI: - logger.info("Extracting features") - hypo_futures = [] - samples = [] - with progress_bar.build_progress_bar(cfg.fairseq.common, itr) as t: - for i, sample in enumerate(t): - if "net_input" not in sample or i < start or i >= end: - continue - if "padding_mask" not in sample["net_input"]: - sample["net_input"]["padding_mask"] = None - - hypos, num_feats = gen_hypos( - generator, models, num_feats, sample, task, use_cuda - ) - hypo_futures.append(hypos) - samples.append(sample) - itr = list(zip(hypo_futures, samples)) - start = 0 - end = len(itr) - logger.info("Finished extracting features") - - with progress_bar.build_progress_bar(cfg.fairseq.common, itr) as t: - for i, sample in enumerate(t): - if i < start or i >= end: - continue - - if hypo_futures is not None: - hypos, sample = sample - hypos = [h.result() for h in hypos] - else: - if "net_input" not in sample: - continue - - hypos, num_feats = gen_hypos( - generator, models, num_feats, sample, task, use_cuda - ) - - for i, sample_id in enumerate(sample["id"].tolist()): - if targets is not None: - target_tokens = targets[sample_id] - elif "target" in sample or "target_label" in sample: - toks = ( - sample["target"][i, :] - if "target_label" not in sample - else sample["target_label"][i, :] - ) - - target_tokens = utils.strip_pad(toks, tgt_dict.pad()).int().cpu() - else: - target_tokens = None - - # Process top predictions - ( - errs, - length_hyp, - length, - hyp_pieces, - hyp_words, - ) = process_predictions( - cfg, - hypos[i], - tgt_dict, - target_tokens, - res_files, - ) - errs_t += errs - lengths_hyp_t += length_hyp - lengths_hyp_unit_t += ( - len(hyp_pieces) if len(hyp_pieces) > 0 else len(hyp_words) - ) - lengths_t += length - count += 1 - all_hyp_pieces.append(hyp_pieces) - all_hyp_words.append(hyp_words) - - num_sentences += ( - sample["nsentences"] if "nsentences" in sample else sample["id"].numel() - ) - - lm_score_sum = 0 - if kenlm is not None: - - if cfg.unit_lm: - lm_score_sum = sum(kenlm.score(w) for w in all_hyp_pieces) - else: - lm_score_sum = sum(kenlm.score(w) for w in all_hyp_words) - elif fairseq_lm is not None: - lm_score_sum = sum(fairseq_lm.score([h.split() for h in all_hyp_words])[0]) - - vt_err_t = 0 - vt_length_t = 0 - if viterbi_transcript is not None: - unit_hyps = [] - if cfg.targets is not None and cfg.lexicon is not None: - lex = {} - with open(cfg.lexicon, "r") as lf: - for line in lf: - items = line.rstrip().split() - lex[items[0]] = items[1:] - for h in all_hyp_pieces: - hyp_ws = [] - for w in h.split(): - assert w in lex, w - hyp_ws.extend(lex[w]) - unit_hyps.append(hyp_ws) - - else: - unit_hyps.extend([h.split() for h in all_hyp_words]) - - vt_err_t = sum( - editdistance.eval(vt, h) for vt, h in zip(viterbi_transcript, unit_hyps) - ) - - vt_length_t = sum(len(h) for h in viterbi_transcript) - - if res_files is not None: - for r in res_files.values(): - r.close() - - gen_timer.stop(lengths_hyp_t) - - return GenResult( - count, - errs_t, - gen_timer, - lengths_hyp_unit_t, - lengths_hyp_t, - lengths_t, - lm_score_sum, - num_feats, - num_sentences, - num_symbols, - vt_err_t, - vt_length_t, - ) - - -def gen_hypos(generator, models, num_feats, sample, task, use_cuda): - sample = utils.move_to_cuda(sample) if use_cuda else sample - - if "features" in sample["net_input"]: - sample["net_input"]["dense_x_only"] = True - num_feats += ( - sample["net_input"]["features"].shape[0] - * sample["net_input"]["features"].shape[1] - ) - hypos = task.inference_step(generator, models, sample, None) - return hypos, num_feats - - -def main(cfg: UnsupGenerateConfig, model=None): - if ( - cfg.fairseq.dataset.max_tokens is None - and cfg.fairseq.dataset.batch_size is None - ): - cfg.fairseq.dataset.max_tokens = 1024000 - - use_cuda = torch.cuda.is_available() and not cfg.fairseq.common.cpu - - task = tasks.setup_task(cfg.fairseq.task) - - overrides = ast.literal_eval(cfg.fairseq.common_eval.model_overrides) - - if cfg.fairseq.task._name == "unpaired_audio_text": - overrides["model"] = { - "blank_weight": cfg.blank_weight, - "blank_mode": cfg.blank_mode, - "blank_is_sil": cfg.sil_is_blank, - "no_softmax": True, - "segmentation": { - "type": "NONE", - }, - } - else: - overrides["model"] = { - "blank_weight": cfg.blank_weight, - "blank_mode": cfg.blank_mode, - } - - if model is None: - # Load ensemble - logger.info("| loading model(s) from {}".format(cfg.fairseq.common_eval.path)) - models, saved_cfg = checkpoint_utils.load_model_ensemble( - cfg.fairseq.common_eval.path.split("\\"), - arg_overrides=overrides, - task=task, - suffix=cfg.fairseq.checkpoint.checkpoint_suffix, - strict=(cfg.fairseq.checkpoint.checkpoint_shard_count == 1), - num_shards=cfg.fairseq.checkpoint.checkpoint_shard_count, - ) - optimize_models(cfg, use_cuda, models) - else: - models = [model] - saved_cfg = cfg.fairseq - - with open_dict(saved_cfg.task): - saved_cfg.task.shuffle = False - saved_cfg.task.sort_by_length = False - - gen_result = generate(cfg, models, saved_cfg, use_cuda) - - wer = None - if gen_result.lengths_t > 0: - wer = gen_result.errs_t * 100.0 / gen_result.lengths_t - logger.info(f"WER: {wer}") - - lm_ppl = float("inf") - - if gen_result.lm_score_t != 0 and gen_result.lengths_hyp_t > 0: - hyp_len = gen_result.lengths_hyp_t - lm_ppl = math.pow( - 10, -gen_result.lm_score_t / (hyp_len + gen_result.num_sentences) - ) - logger.info(f"LM PPL: {lm_ppl}") - - logger.info( - "| Processed {} sentences ({} tokens) in {:.1f}s ({:.2f}" - " sentences/s, {:.2f} tokens/s)".format( - gen_result.num_sentences, - gen_result.gen_timer.n, - gen_result.gen_timer.sum, - gen_result.num_sentences / gen_result.gen_timer.sum, - 1.0 / gen_result.gen_timer.avg, - ) - ) - - vt_diff = None - if gen_result.vt_length_t > 0: - vt_diff = gen_result.vt_err_t / gen_result.vt_length_t - vt_diff = max(cfg.min_vt_uer, vt_diff) - - lm_ppl = max(cfg.min_lm_ppl, lm_ppl) - - if not cfg.unsupervised_tuning: - weighted_score = wer - else: - weighted_score = math.log(lm_ppl) * (vt_diff or 1.0) - - res = ( - f"| Generate {cfg.fairseq.dataset.gen_subset} with beam={cfg.beam}, " - f"lm_weight={cfg.kaldi_decoder_config.acoustic_scale if cfg.kaldi_decoder_config else cfg.lm_weight}, " - f"word_score={cfg.word_score}, sil_weight={cfg.sil_weight}, blank_weight={cfg.blank_weight}, " - f"WER: {wer}, LM_PPL: {lm_ppl}, num feats: {gen_result.num_feats}, " - f"length: {gen_result.lengths_hyp_t}, UER to viterbi: {(vt_diff or 0) * 100}, score: {weighted_score}" - ) - - logger.info(res) - # print(res) - - return task, weighted_score - - -@hydra.main( - config_path=os.path.join("../../..", "fairseq", "config"), config_name="config" -) -def hydra_main(cfg): - with open_dict(cfg): - # make hydra logging work with ddp (see # see https://github.com/facebookresearch/hydra/issues/1126) - cfg.job_logging_cfg = OmegaConf.to_container( - HydraConfig.get().job_logging, resolve=True - ) - - cfg = OmegaConf.create( - OmegaConf.to_container(cfg, resolve=False, enum_to_str=False) - ) - OmegaConf.set_struct(cfg, True) - logger.info(cfg) - - utils.import_user_module(cfg.fairseq.common) - - _, score = main(cfg) - - if cfg.is_ax: - return score, None - return score - - -def cli_main(): - try: - from hydra._internal.utils import get_args - - cfg_name = get_args().config_name or "config" - except: - logger.warning("Failed to get config name from hydra args") - cfg_name = "config" - - cs = ConfigStore.instance() - cs.store(name=cfg_name, node=UnsupGenerateConfig) - hydra_main() - - -if __name__ == "__main__": - cli_main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/vq-wav2vec_featurize.py b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/vq-wav2vec_featurize.py deleted file mode 100644 index 54d06c028482f10e4eb743df23717e68b35141e1..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/vq-wav2vec_featurize.py +++ /dev/null @@ -1,249 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -""" -Helper script to pre-compute embeddings for a flashlight (previously called wav2letter++) dataset -""" - -import argparse -import glob -import os -import os.path as osp -import pprint - -import fairseq -import soundfile as sf -import torch -from torch import nn -from torch.utils.data import DataLoader - -try: - import tqdm -except: - print("Install tqdm to use --log-format=tqdm") - - -class FilesDataset: - def __init__(self, files, labels): - self.files = files - if labels and osp.exists(labels): - with open(labels, "r") as lbl_f: - self.labels = [line.rstrip() for line in lbl_f] - else: - self.labels = labels - - def __len__(self): - return len(self.files) - - def __getitem__(self, index): - fname = self.files[index] - - wav, sr = sf.read(fname) - assert sr == 16000 - - wav = torch.from_numpy(wav).float() - lbls = None - if self.labels: - if isinstance(self.labels, str): - lbl_file = osp.splitext(fname)[0] + "." + self.labels - with open(lbl_file, "r") as lblf: - lbls = lblf.readline() - assert lbls is not None - else: - lbls = self.labels[index] - return wav, lbls - - def collate(self, batch): - return batch - - -class ArgTypes: - @staticmethod - def existing_path(arg): - arg = str(arg) - assert osp.exists(arg), f"File {arg} does not exist" - return arg - - @staticmethod - def mkdir(arg): - arg = str(arg) - os.makedirs(arg, exist_ok=True) - return arg - - -class DatasetWriter: - def __init__(self): - - self.args = self.load_config() - pprint.pprint(self.args.__dict__) - - self.model = self.load_model() - - def __getattr__(self, attr): - return getattr(self.args, attr) - - def read_manifest(self, fname): - - with open(fname, "r") as fp: - lines = fp.read().split("\n") - root = lines.pop(0).strip() - fnames = [ - osp.join(root, line.split("\t")[0]) for line in lines if len(line) > 0 - ] - - return fnames - - def process_splits(self): - - if self.args.shard is not None or self.args.num_shards is not None: - assert self.args.shard is not None and self.args.num_shards is not None - - for split in self.splits: - print(split) - - if self.extension == "tsv": - datadir = osp.join(self.data_dir, f"{split}.{self.extension}") - print("Reading manifest file: ", datadir) - files = self.read_manifest(datadir) - else: - datadir = osp.join(self.data_dir, split, f"**/*.{self.extension}") - files = glob.glob(datadir, recursive=True) - - assert len(files) > 0 - - if self.args.shard is not None: - files = files[self.args.shard :: self.args.num_shards] - - lbls = [] - with open(self.data_file(split), "w") as srcf: - for line, lbl in self.iterate(files): - print(line, file=srcf) - if self.args.labels: - lbls.append(lbl + "\n") - - if self.args.labels: - assert all(a is not None for a in lbls) - with open(self.lbl_file(split), "w") as lblf: - lblf.writelines(lbls) - - def iterate(self, files): - - data = self.load_data(files) - for samples in tqdm.tqdm(data, total=len(files) // 32): - - for wav, lbl in samples: - x = wav.unsqueeze(0).float().cuda() - - div = 1 - while x.size(-1) // div > self.args.max_size: - div += 1 - - xs = x.chunk(div, dim=-1) - - result = [] - for x in xs: - torch.cuda.empty_cache() - x = self.model.feature_extractor(x) - if self.quantize_location == "encoder": - with torch.no_grad(): - _, idx = self.model.vector_quantizer.forward_idx(x) - idx = idx.squeeze(0).cpu() - else: - with torch.no_grad(): - z = self.model.feature_aggregator(x) - _, idx = self.model.vector_quantizer.forward_idx(z) - idx = idx.squeeze(0).cpu() - result.append(idx) - - idx = torch.cat(result, dim=0) - yield " ".join("-".join(map(str, a.tolist())) for a in idx), lbl - - def lbl_file(self, name): - shard_part = "" if self.args.shard is None else f".{self.args.shard}" - return osp.join(self.output_dir, f"{name}.lbl{shard_part}") - - def data_file(self, name): - shard_part = "" if self.args.shard is None else f".{self.args.shard}" - return osp.join(self.output_dir, f"{name}.src{shard_part}") - - def var_file(self): - return osp.join(self.output_dir, f"vars.pt") - - def load_config(self): - - parser = argparse.ArgumentParser("Vector Quantized wav2vec features") - - # Model Arguments - parser.add_argument("--checkpoint", type=ArgTypes.existing_path, required=True) - parser.add_argument("--data-parallel", action="store_true") - - # Output Arguments - parser.add_argument("--output-dir", type=ArgTypes.mkdir, required=True) - - # Data Arguments - parser.add_argument("--data-dir", type=ArgTypes.existing_path, required=True) - parser.add_argument("--splits", type=str, nargs="+", required=True) - parser.add_argument("--extension", type=str, required=True) - parser.add_argument("--labels", type=str, required=False) - - parser.add_argument("--shard", type=int, default=None) - parser.add_argument("--num-shards", type=int, default=None) - parser.add_argument("--max-size", type=int, default=1300000) - - # Logger Arguments - parser.add_argument( - "--log-format", type=str, choices=["none", "simple", "tqdm"] - ) - - return parser.parse_args() - - def load_data(self, fnames): - - dataset = FilesDataset(fnames, self.args.labels) - loader = DataLoader( - dataset, batch_size=32, collate_fn=dataset.collate, num_workers=8 - ) - return loader - - def load_model(self): - model, cfg, task = fairseq.checkpoint_utils.load_model_ensemble_and_task([self.checkpoint]) - model = model[0] - - self.quantize_location = getattr(cfg.model, "vq", "encoder") - - model.eval().float() - model.cuda() - - if self.data_parallel: - model = nn.DataParallel(model) - - return model - - def __call__(self): - - self.process_splits() - - if hasattr(self.model.feature_extractor, "vars") and ( - self.args.shard is None or self.args.shard == 0 - ): - vars = ( - self.model.feature_extractor.vars.view( - self.model.feature_extractor.banks, - self.model.feature_extractor.num_vars, - -1, - ) - .cpu() - .detach() - ) - print("writing learned latent variable embeddings: ", vars.shape) - torch.save(vars, self.var_file()) - - -if __name__ == "__main__": - write_data = DatasetWriter() - - write_data() - print("Done.") diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/wav2vec_featurize.py b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/wav2vec_featurize.py deleted file mode 100644 index 49374761f5ff6b23ed6aa05abd2bc14a145b9c07..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/wav2vec_featurize.py +++ /dev/null @@ -1,249 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -""" -Helper script to pre-compute embeddings for a flashlight (previously called wav2letter++) dataset -""" - -import argparse -import glob -import os -from shutil import copy - -import fairseq -import h5py -import numpy as np -import soundfile as sf -import torch -import tqdm -from torch import nn - - -def read_audio(fname): - """ Load an audio file and return PCM along with the sample rate """ - - wav, sr = sf.read(fname) - assert sr == 16e3 - - return wav, 16e3 - - -class PretrainedWav2VecModel(nn.Module): - def __init__(self, fname): - super().__init__() - - model, cfg, task = fairseq.checkpoint_utils.load_model_ensemble_and_task([fname]) - model = model[0] - model.eval() - - self.model = model - - def forward(self, x): - with torch.no_grad(): - z = self.model.feature_extractor(x) - if isinstance(z, tuple): - z = z[0] - c = self.model.feature_aggregator(z) - return z, c - - -class EmbeddingWriterConfig(argparse.ArgumentParser): - def __init__(self): - super().__init__("Pre-compute embeddings for flashlight datasets") - - kwargs = {"action": "store", "type": str, "required": True} - - self.add_argument("--input", "-i", help="Input Directory", **kwargs) - self.add_argument("--output", "-o", help="Output Directory", **kwargs) - self.add_argument("--model", help="Path to model checkpoint", **kwargs) - self.add_argument("--split", help="Dataset Splits", nargs="+", **kwargs) - self.add_argument( - "--ext", default="wav", required=False, help="Audio file extension" - ) - - self.add_argument( - "--no-copy-labels", - action="store_true", - help="Do not copy label files. Useful for large datasets, use --targetdir in flashlight then.", - ) - self.add_argument( - "--use-feat", - action="store_true", - help="Use the feature vector ('z') instead of context vector ('c') for features", - ) - self.add_argument("--gpu", help="GPU to use", default=0, type=int) - - -class Prediction: - """ Lightweight wrapper around a fairspeech embedding model """ - - def __init__(self, fname, gpu=0): - self.gpu = gpu - self.model = PretrainedWav2VecModel(fname).cuda(gpu) - - def __call__(self, x): - x = torch.from_numpy(x).float().cuda(self.gpu) - with torch.no_grad(): - z, c = self.model(x.unsqueeze(0)) - - return z.squeeze(0).cpu().numpy(), c.squeeze(0).cpu().numpy() - - -class H5Writer: - """ Write features as hdf5 file in flashlight compatible format """ - - def __init__(self, fname): - self.fname = fname - os.makedirs(os.path.dirname(self.fname), exist_ok=True) - - def write(self, data): - channel, T = data.shape - - with h5py.File(self.fname, "w") as out_ds: - data = data.T.flatten() - out_ds["features"] = data - out_ds["info"] = np.array([16e3 // 160, T, channel]) - - -class EmbeddingDatasetWriter(object): - """Given a model and a flashlight dataset, pre-compute and store embeddings - - Args: - input_root, str : - Path to the flashlight dataset - output_root, str : - Desired output directory. Will be created if non-existent - split, str : - Dataset split - """ - - def __init__( - self, - input_root, - output_root, - split, - model_fname, - extension="wav", - gpu=0, - verbose=False, - use_feat=False, - ): - - assert os.path.exists(model_fname) - - self.model_fname = model_fname - self.model = Prediction(self.model_fname, gpu) - - self.input_root = input_root - self.output_root = output_root - self.split = split - self.verbose = verbose - self.extension = extension - self.use_feat = use_feat - - assert os.path.exists(self.input_path), "Input path '{}' does not exist".format( - self.input_path - ) - - def _progress(self, iterable, **kwargs): - if self.verbose: - return tqdm.tqdm(iterable, **kwargs) - return iterable - - def require_output_path(self, fname=None): - path = self.get_output_path(fname) - os.makedirs(path, exist_ok=True) - - @property - def input_path(self): - return self.get_input_path() - - @property - def output_path(self): - return self.get_output_path() - - def get_input_path(self, fname=None): - if fname is None: - return os.path.join(self.input_root, self.split) - return os.path.join(self.get_input_path(), fname) - - def get_output_path(self, fname=None): - if fname is None: - return os.path.join(self.output_root, self.split) - return os.path.join(self.get_output_path(), fname) - - def copy_labels(self): - self.require_output_path() - - labels = list( - filter( - lambda x: self.extension not in x, glob.glob(self.get_input_path("*")) - ) - ) - for fname in tqdm.tqdm(labels): - copy(fname, self.output_path) - - @property - def input_fnames(self): - return sorted(glob.glob(self.get_input_path("*.{}".format(self.extension)))) - - def __len__(self): - return len(self.input_fnames) - - def write_features(self): - - paths = self.input_fnames - - fnames_context = map( - lambda x: os.path.join( - self.output_path, x.replace("." + self.extension, ".h5context") - ), - map(os.path.basename, paths), - ) - - for name, target_fname in self._progress( - zip(paths, fnames_context), total=len(self) - ): - wav, sr = read_audio(name) - z, c = self.model(wav) - feat = z if self.use_feat else c - writer = H5Writer(target_fname) - writer.write(feat) - - def __repr__(self): - - return "EmbeddingDatasetWriter ({n_files} files)\n\tinput:\t{input_root}\n\toutput:\t{output_root}\n\tsplit:\t{split})".format( - n_files=len(self), **self.__dict__ - ) - - -if __name__ == "__main__": - - args = EmbeddingWriterConfig().parse_args() - - for split in args.split: - - writer = EmbeddingDatasetWriter( - input_root=args.input, - output_root=args.output, - split=split, - model_fname=args.model, - gpu=args.gpu, - extension=args.ext, - use_feat=args.use_feat, - ) - - print(writer) - writer.require_output_path() - - print("Writing Features...") - writer.write_features() - print("Done.") - - if not args.no_copy_labels: - print("Copying label data...") - writer.copy_labels() - print("Done.") diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/wav2vec_manifest.py b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/wav2vec_manifest.py deleted file mode 100644 index 9b8aa180e88d9ee98bdca7089aed5046ec0d9cb9..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/wav2vec_manifest.py +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. -""" -Data pre-processing: build vocabularies and binarize training data. -""" - -import argparse -import glob -import os -import random - -import soundfile - - -def get_parser(): - parser = argparse.ArgumentParser() - parser.add_argument( - "root", metavar="DIR", help="root directory containing flac files to index" - ) - parser.add_argument( - "--valid-percent", - default=0.01, - type=float, - metavar="D", - help="percentage of data to use as validation set (between 0 and 1)", - ) - parser.add_argument( - "--dest", default=".", type=str, metavar="DIR", help="output directory" - ) - parser.add_argument( - "--ext", default="flac", type=str, metavar="EXT", help="extension to look for" - ) - parser.add_argument("--seed", default=42, type=int, metavar="N", help="random seed") - parser.add_argument( - "--path-must-contain", - default=None, - type=str, - metavar="FRAG", - help="if set, path must contain this substring for a file to be included in the manifest", - ) - return parser - - -def main(args): - assert args.valid_percent >= 0 and args.valid_percent <= 1.0 - - if not os.path.exists(args.dest): - os.makedirs(args.dest) - - dir_path = os.path.realpath(args.root) - search_path = os.path.join(dir_path, "**/*." + args.ext) - rand = random.Random(args.seed) - - valid_f = ( - open(os.path.join(args.dest, "valid.tsv"), "w") - if args.valid_percent > 0 - else None - ) - - with open(os.path.join(args.dest, "train.tsv"), "w") as train_f: - print(dir_path, file=train_f) - - if valid_f is not None: - print(dir_path, file=valid_f) - - for fname in glob.iglob(search_path, recursive=True): - file_path = os.path.realpath(fname) - - if args.path_must_contain and args.path_must_contain not in file_path: - continue - - frames = soundfile.info(fname).frames - dest = train_f if rand.random() > args.valid_percent else valid_f - print( - "{}\t{}".format(os.path.relpath(file_path, dir_path), frames), file=dest - ) - if valid_f is not None: - valid_f.close() - - -if __name__ == "__main__": - parser = get_parser() - args = parser.parse_args() - main(args) diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/xlsr/README.md b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/xlsr/README.md deleted file mode 100644 index 84e9de36a6e6b9d94e421841b3957dca4b6c7234..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/xlsr/README.md +++ /dev/null @@ -1,68 +0,0 @@ -# XLS-R - -XLS-R is a set of large-scale models for self-supervised cross-lingual speech representation learning based on wav2vec 2.0. It was pretrained on 128 languages and approximately 436K hours of unlabeled speech data. With finetuning, these models achieve state of the art performance in speech translation, speech recognition and language identification. We evaluate the model across multiple benchmarks such as CoVoST-2 for speech translation, BABEL / MLS / CommonVoice / VoxPopuli for automatic speech recognition, and VoxLingua107 for language identification as we llas VoxCeleb1 for speaker identification. More details about this work can be found in our [paper](https://arxiv.org/pdf/2111.09296.pdf) and download links can be found below. - -Model | Link -|------|------ -XLS-R 300M | [download](https://dl.fbaipublicfiles.com/fairseq/wav2vec/xlsr2_300m.pt) -XLS-R 1B | [download](https://dl.fbaipublicfiles.com/fairseq/wav2vec/xlsr2_960m_1000k.pt) -XLS-R 2B | [download](https://dl.fbaipublicfiles.com/fairseq/wav2vec/xlsr2_2B_1000k.pt) - -You can also download these models [here](https://huggingface.co/models?other=xls_r) and read more about it in the [blogpost](https://huggingface.co/blog/fine-tune-xlsr-wav2vec2) from Hugging Face. - -## Speech Translation Finetuned Models - -We multilingually finetune XLS-R models on [CoVoST 2](https://github.com/facebookresearch/covost), which has 21 -into-English and 15 out-of-English directions. - -Model | Directions | Link -|------|------|------ -XLS-R 300M | 21 langs → En | [download](https://dl.fbaipublicfiles.com/fairseq/wav2vec/xls_r_300m_21_en.pt) -XLS-R 300M | En → 15 langs | [download](https://dl.fbaipublicfiles.com/fairseq/wav2vec/xls_r_300m_en_15.pt) -XLS-R 1B | 21 langs → En | [download](https://dl.fbaipublicfiles.com/fairseq/wav2vec/xls_r_1b_21_en.pt) -XLS-R 1B | En → 15 langs | [download](https://dl.fbaipublicfiles.com/fairseq/wav2vec/xls_r_1b_en_15.pt) -XLS-R 2B | 21 langs → En | [download](https://dl.fbaipublicfiles.com/fairseq/wav2vec/xls_r_2b_21_en.pt) -XLS-R 2B | En → 15 langs | [download](https://dl.fbaipublicfiles.com/fairseq/wav2vec/xls_r_2b_en_15.pt) -XLS-R 2B | 21 langs → En + En → 15 langs | [download](https://dl.fbaipublicfiles.com/fairseq/wav2vec/xls_r_2b_22_16.pt) - -## ASR Finetuning - -You can refer the original wav2vec documentation on detailed instructions about how to finetune a pretrained model with CTC [here](https://github.com/pytorch/fairseq/tree/main/examples/wav2vec#fine-tune-a-pre-trained-model-with-ctc). Below is an example command and you can find the values for different hyperparameters to reproduce the results in our paper. - -```shell script -$ fairseq-hydra-train \ - distributed_training.distributed_port=$PORT \ - task.data=/path/to/data \ - model.w2v_path=/path/to/model.pt \ - --config-dir /path/to/fairseq-py/examples/wav2vec/xlsr/config \ - --config-name finetune -``` - -For finetuning the 300M as well as 1B model, we use the same hyperparameter setting defined in `finetune.yaml`. We vary `optimization.max_update` as described in the below table and the `optimization.lr` is picked from the interval [2e-5, 3e-4] based on dev word error rate. - -Benchmark | Total Number of Updates -|------|------ -Babel | 26000 -Common Voice | 13000 -VoxPopuli | 50000 -MLS 10h | 20000 - -For finetuning the 2B model, we make some additional changes for `finetune.yaml` . We use the fully_sharded `distributed_training.ddp_backend` provided by the [fairscale](https://github.com/facebookresearch/fairscale) library and and set `model.activation_checkpoint` to true. We also increase `dataset.max_tokens` to 2560000 and use a total effective batch size of 2560000*24. We sweep for the best `optimization.lr` within the interval [3e−6,3e−5] using dev error rate. For common voice dataset, we pick the `model.mask_prob` for different languages among {0.30, 0.40} based on best dev error rate. - - - -## Citation - -Please cite as: - -``` bibtex -@article{babu2021xlsr, - title={XLS-R: Self-supervised Cross-lingual Speech Representation Learning at Scale}, - author={Arun Babu and Changhan Wang and Andros Tjandra and Kushal Lakhotia and Qiantong Xu and Naman Goyal and Kritika Singh and Patrick von Platen and Yatharth Saraf and Juan Pino and Alexei Baevski and Alexis Conneau and Michael Auli}, - year={2021}, - volume={abs/2111.09296}, - journal={arXiv}, -} -``` - - diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/xlsr/config/finetune.yaml b/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/xlsr/config/finetune.yaml deleted file mode 100644 index 8736e101c5047f74ee24c6198f0f9523c9e9eef6..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wav2vec/xlsr/config/finetune.yaml +++ /dev/null @@ -1,66 +0,0 @@ -# @package _group_ - -common: - fp16: true - log_format: json - log_interval: 200 - tensorboard_logdir: tb - -checkpoint: - save_interval: 1000 - save_interval_updates: 1000 - keep_interval_updates: 1 - no_epoch_checkpoints: true - best_checkpoint_metric: wer - -task: - _name: audio_finetuning - data: ??? - normalize: true - labels: ltr - -dataset: - num_workers: 6 - max_tokens: 1280000 - skip_invalid_size_inputs_valid_test: true - validate_after_updates: 10000 - validate_interval_updates: 1000 - valid_subset: valid - -distributed_training: - ddp_backend: legacy_ddp - distributed_world_size: 4 - -criterion: - _name: ctc - zero_infinity: true - -optimization: - max_update: ??? - lr: [0.0003] - sentence_avg: true - update_freq: [5] - -optimizer: - _name: adam - adam_betas: (0.9,0.98) - adam_eps: 1e-08 - -lr_scheduler: - _name: tri_stage - phase_ratio: [0.1, 0.4, 0.5] - final_lr_scale: 0.05 - -model: - _name: wav2vec_ctc - w2v_path: ??? - apply_mask: true - mask_prob: 0.75 - mask_channel_prob: 0.25 - mask_channel_length: 64 - layerdrop: 0.1 - activation_dropout: 0.1 - feature_grad_mult: 0.0 - freeze_finetune_updates: 10000 - - checkpoint_activations: false diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wmt19/README.md b/audio/safeear/safeear_repo/fairseq_ours/examples/wmt19/README.md deleted file mode 100644 index 5c90d0e6c4ae8d043ca622e70c5828dca6f9c2f2..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wmt19/README.md +++ /dev/null @@ -1,85 +0,0 @@ -# WMT 19 - -This page provides pointers to the models of Facebook-FAIR's WMT'19 news translation task submission [(Ng et al., 2019)](https://arxiv.org/abs/1907.06616). - -## Pre-trained models - -Model | Description | Download ----|---|--- -`transformer.wmt19.en-de` | En->De Ensemble | [download (.tar.gz)](https://dl.fbaipublicfiles.com/fairseq/models/wmt19.en-de.joined-dict.ensemble.tar.gz) -`transformer.wmt19.de-en` | De->En Ensemble | [download (.tar.gz)](https://dl.fbaipublicfiles.com/fairseq/models/wmt19.de-en.joined-dict.ensemble.tar.gz) -`transformer.wmt19.en-ru` | En->Ru Ensemble | [download (.tar.gz)](https://dl.fbaipublicfiles.com/fairseq/models/wmt19.en-ru.ensemble.tar.gz) -`transformer.wmt19.ru-en` | Ru->En Ensemble | [download (.tar.gz)](https://dl.fbaipublicfiles.com/fairseq/models/wmt19.ru-en.ensemble.tar.gz) -`transformer_lm.wmt19.en` | En Language Model | [download (.tar.gz)](https://dl.fbaipublicfiles.com/fairseq/models/lm/wmt19.en.tar.gz) -`transformer_lm.wmt19.de` | De Language Model | [download (.tar.gz)](https://dl.fbaipublicfiles.com/fairseq/models/lm/wmt19.de.tar.gz) -`transformer_lm.wmt19.ru` | Ru Language Model | [download (.tar.gz)](https://dl.fbaipublicfiles.com/fairseq/models/lm/wmt19.ru.tar.gz) - -## Pre-trained single models before finetuning - -Model | Description | Download ----|---|--- -`transformer.wmt19.en-de` | En->De Single, no finetuning | [download (.tar.gz)](https://dl.fbaipublicfiles.com/fairseq/models/wmt19.en-de.ffn8192.tar.gz) -`transformer.wmt19.de-en` | De->En Single, no finetuning | [download (.tar.gz)](https://dl.fbaipublicfiles.com/fairseq/models/wmt19.de-en.ffn8192.tar.gz) -`transformer.wmt19.en-ru` | En->Ru Single, no finetuning | [download (.tar.gz)](https://dl.fbaipublicfiles.com/fairseq/models/wmt19.en-ru.ffn8192.tar.gz) -`transformer.wmt19.ru-en` | Ru->En Single, no finetuning | [download (.tar.gz)](https://dl.fbaipublicfiles.com/fairseq/models/wmt19.ru-en.ffn8192.tar.gz) - -## Example usage (torch.hub) - -#### Requirements - -We require a few additional Python dependencies for preprocessing: -```bash -pip install fastBPE sacremoses -``` - -#### Translation - -```python -import torch - -# English to German translation -en2de = torch.hub.load('pytorch/fairseq', 'transformer.wmt19.en-de', checkpoint_file='model1.pt:model2.pt:model3.pt:model4.pt', - tokenizer='moses', bpe='fastbpe') -en2de.translate("Machine learning is great!") # 'Maschinelles Lernen ist großartig!' - -# German to English translation -de2en = torch.hub.load('pytorch/fairseq', 'transformer.wmt19.de-en', checkpoint_file='model1.pt:model2.pt:model3.pt:model4.pt', - tokenizer='moses', bpe='fastbpe') -de2en.translate("Maschinelles Lernen ist großartig!") # 'Machine learning is great!' - -# English to Russian translation -en2ru = torch.hub.load('pytorch/fairseq', 'transformer.wmt19.en-ru', checkpoint_file='model1.pt:model2.pt:model3.pt:model4.pt', - tokenizer='moses', bpe='fastbpe') -en2ru.translate("Machine learning is great!") # 'Машинное обучение - это здорово!' - -# Russian to English translation -ru2en = torch.hub.load('pytorch/fairseq', 'transformer.wmt19.ru-en', checkpoint_file='model1.pt:model2.pt:model3.pt:model4.pt', - tokenizer='moses', bpe='fastbpe') -ru2en.translate("Машинное обучение - это здорово!") # 'Machine learning is great!' -``` - -#### Language Modeling - -```python -# Sample from the English LM -en_lm = torch.hub.load('pytorch/fairseq', 'transformer_lm.wmt19.en', tokenizer='moses', bpe='fastbpe') -en_lm.sample("Machine learning is") # 'Machine learning is the future of computing, says Microsoft boss Satya Nadella ...' - -# Sample from the German LM -de_lm = torch.hub.load('pytorch/fairseq', 'transformer_lm.wmt19.de', tokenizer='moses', bpe='fastbpe') -de_lm.sample("Maschinelles lernen ist") # 'Maschinelles lernen ist das A und O (neues-deutschland.de) Die Arbeitsbedingungen für Lehrerinnen und Lehrer sind seit Jahren verbesserungswürdig ...' - -# Sample from the Russian LM -ru_lm = torch.hub.load('pytorch/fairseq', 'transformer_lm.wmt19.ru', tokenizer='moses', bpe='fastbpe') -ru_lm.sample("машинное обучение это") # 'машинное обучение это то, что мы называем "искусственным интеллектом".' -``` - -## Citation -```bibtex -@inproceedings{ng2019facebook}, - title = {Facebook FAIR's WMT19 News Translation Task Submission}, - author = {Ng, Nathan and Yee, Kyra and Baevski, Alexei and Ott, Myle and Auli, Michael and Edunov, Sergey}, - booktitle = {Proc. of WMT}, - year = 2019, -} -``` diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wmt20/README.md b/audio/safeear/safeear_repo/fairseq_ours/examples/wmt20/README.md deleted file mode 100644 index b4f2874652f8be19998a65faa1d9276d8017ec59..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wmt20/README.md +++ /dev/null @@ -1,72 +0,0 @@ -# WMT 20 - -This page provides pointers to the models of Facebook-FAIR's WMT'20 news translation task submission [(Chen et al., 2020)](https://arxiv.org/abs/2011.08298). - -## Single best MT models (after finetuning on part of WMT20 news dev set) - -Model | Description | Download ----|---|--- -`transformer.wmt20.ta-en` | Ta->En | [download (.tar.gz)](https://dl.fbaipublicfiles.com/fairseq/models/wmt20.ta-en.single.tar.gz) -`transformer.wmt20.en-ta` | En->Ta | [download (.tar.gz)](https://dl.fbaipublicfiles.com/fairseq/models/wmt20.en-ta.single.tar.gz) -`transformer.wmt20.iu-en.news` | Iu->En (News domain) | [download (.tar.gz)](https://dl.fbaipublicfiles.com/fairseq/models/wmt20.iu-en.news.single.tar.gz) -`transformer.wmt20.en-iu.news` | En->Iu (News domain) | [download (.tar.gz)](https://dl.fbaipublicfiles.com/fairseq/models/wmt20.en-iu.news.single.tar.gz) -`transformer.wmt20.iu-en.nh` | Iu->En (Nunavut Hansard domain) | [download (.tar.gz)](https://dl.fbaipublicfiles.com/fairseq/models/wmt20.iu-en.nh.single.tar.gz) -`transformer.wmt20.en-iu.nh` | En->Iu (Nunavut Hansard domain) | [download (.tar.gz)](https://dl.fbaipublicfiles.com/fairseq/models/wmt20.en-iu.nh.single.tar.gz) - -## Language models -Model | Description | Download ----|---|--- -`transformer_lm.wmt20.en` | En Language Model | [download (.tar.gz)](https://dl.fbaipublicfiles.com/fairseq/models/wmt20.en.tar.gz) -`transformer_lm.wmt20.ta` | Ta Language Model | [download (.tar.gz)](https://dl.fbaipublicfiles.com/fairseq/models/wmt20.ta.tar.gz) -`transformer_lm.wmt20.iu.news` | Iu Language Model (News domain) | [download (.tar.gz)](https://dl.fbaipublicfiles.com/fairseq/models/wmt20.iu.news.tar.gz) -`transformer_lm.wmt20.iu.nh` | Iu Language Model (Nunavut Hansard domain) | [download (.tar.gz)](https://dl.fbaipublicfiles.com/fairseq/models/wmt20.iu.nh.tar.gz) - -## Example usage (torch.hub) - -#### Translation - -```python -import torch - -# English to Tamil translation -en2ta = torch.hub.load('pytorch/fairseq', 'transformer.wmt20.en-ta') -en2ta.translate("Machine learning is great!") # 'இயந்திரக் கற்றல் அருமை!' - -# Tamil to English translation -ta2en = torch.hub.load('pytorch/fairseq', 'transformer.wmt20.ta-en') -ta2en.translate("இயந்திரக் கற்றல் அருமை!") # 'Machine learning is great!' - -# English to Inuktitut translation -en2iu = torch.hub.load('pytorch/fairseq', 'transformer.wmt20.en-iu.news') -en2iu.translate("machine learning is great!") # 'ᖃᒧᑕᐅᔭᓄᑦ ᐃᓕᓐᓂᐊᕐᓂᖅ ᐱᐅᔪᒻᒪᕆᒃ!' - -# Inuktitut to English translation -iu2en = torch.hub.load('pytorch/fairseq', 'transformer.wmt20.iu-en.news') -iu2en.translate("ᖃᒧᑕᐅᔭᓄᑦ ᐃᓕᓐᓂᐊᕐᓂᖅ ᐱᐅᔪᒻᒪᕆᒃ!") # 'Machine learning excellence!' -``` - -#### Language Modeling - -```python -# Sample from the English LM -en_lm = torch.hub.load('pytorch/fairseq', 'transformer_lm.wmt20.en') -en_lm.sample("Machine learning is") # 'Machine learning is a type of artificial intelligence that uses machine learning to learn from data and make predictions.' - -# Sample from the Tamil LM -ta_lm = torch.hub.load('pytorch/fairseq', 'transformer_lm.wmt20.ta') -ta_lm.sample("இயந்திரக் கற்றல் என்பது செயற்கை நுண்ணறிவின்") # 'இயந்திரக் கற்றல் என்பது செயற்கை நுண்ணறிவின் ஒரு பகுதியாகும்.' - -# Sample from the Inuktitut LM -iu_lm = torch.hub.load('pytorch/fairseq', 'transformer_lm.wmt20.iu.news') -iu_lm.sample("ᖃᒧᑕᐅᔭᓄᑦ ᐃᓕᓐᓂᐊᕐᓂᖅ") # 'ᖃᒧᑕᐅᔭᓄᑦ ᐃᓕᓐᓂᐊᕐᓂᖅ, ᐊᒻᒪᓗ ᓯᓚᐅᑉ ᐊᓯᙳᖅᐸᓪᓕᐊᓂᖓᓄᑦ ᖃᓄᐃᓕᐅᕈᑎᒃᓴᑦ, ᐃᓚᖃᖅᖢᑎᒃ ᐅᑯᓂᖓ:' -``` - -## Citation -```bibtex -@inproceedings{chen2020facebook - title={Facebook AI's WMT20 News Translation Task Submission}, - author={Peng-Jen Chen and Ann Lee and Changhan Wang and Naman Goyal and Angela Fan and Mary Williamson and Jiatao Gu}, - booktitle={Proc. of WMT}, - year={2020}, -} -``` diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wmt21/README.md b/audio/safeear/safeear_repo/fairseq_ours/examples/wmt21/README.md deleted file mode 100644 index 524fffb7247bce6ab2f54e2124bcc3d680702778..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wmt21/README.md +++ /dev/null @@ -1,25 +0,0 @@ -# WMT 21 - -This page provides pointers to the models of Facebook AI's WMT'21 news translation task submission [(Tran et al., 2021)](https://arxiv.org/abs/2108.03265). - -## Single best dense models - -Model | Description | Download ----|---|--- -`wmt21.dense-24-wide.X-En` | X-En | [download (.tar.gz)](https://dl.fbaipublicfiles.com/fairseq/models/wmt21.dense-24-wide.X-En.tar.gz) -`wmt21.dense-24-wide.En-X` | En-X | [download (.tar.gz)](https://dl.fbaipublicfiles.com/fairseq/models/wmt21.dense-24-wide.En-X.tar.gz) - -## Example usage - -See eval.sh - - -## Citation -```bibtex -@inproceedings{tran2021facebook - title={Facebook AI’s WMT21 News Translation Task Submission}, - author={Chau Tran and Shruti Bhosale and James Cross and Philipp Koehn and Sergey Edunov and Angela Fan}, - booktitle={Proc. of WMT}, - year={2021}, -} -``` diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wmt21/eval.sh b/audio/safeear/safeear_repo/fairseq_ours/examples/wmt21/eval.sh deleted file mode 100644 index b36d934c51b948a54f11a62cbc486a17431eea89..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wmt21/eval.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash -# Copyright (c) Facebook, Inc. and its affiliates. -# All rights reserved. -# -# This source code is licensed under the license found in the -# LICENSE file in the root directory of this source tree. -SRC=en -TGT=is -MODEL_NAME=wmt21.dense-24-wide.En-X - -PATH_TO_FAIRSEQ_PY=. -TMP_DIR=generation_tmp -mkdir -p $TMP_DIR - -REPLACE_UNICODE_PUNCT=$PATH_TO_FAIRSEQ_PY/examples/wmt21/scripts/replace-unicode-punctuation.perl -NORM_PUNCT=$PATH_TO_FAIRSEQ_PY/examples/wmt21/scripts/normalize-punctuation.perl -if [ ! -d "${TMP_DIR}/${MODEL_NAME}" ]; then - wget https://dl.fbaipublicfiles.com/fairseq/models/${MODEL_NAME}.tar.gz -P $TMP_DIR/ - tar -xvf $TMP_DIR/${MODEL_NAME}.tar.gz -C $TMP_DIR -fi -MODEL_DIR=$TMP_DIR/${MODEL_NAME} -if [ ! -d "${TMP_DIR}/wmt21-news-systems" ]; then - git clone https://github.com/wmt-conference/wmt21-news-systems $TMP_DIR/wmt21-news-systems -fi - -DOMAIN_TAG="wmtdata newsdomain" -INPUT_FILE=$TMP_DIR/wmt21-news-systems/txt/sources/newstest2021.${SRC}-${TGT}.src.${SRC} -REF_FILE=$TMP_DIR/wmt21-news-systems/txt/references/newstest2021.${SRC}-${TGT}.ref.A.${TGT} - -# Translate -cat ${INPUT_FILE} | sed "s/^/${DOMAIN_TAG} /" | $REPLACE_UNICODE_PUNCT | $NORM_PUNCT -l ${SRC} | python $PATH_TO_FAIRSEQ_PY/fairseq_cli/interactive.py $MODEL_DIR \ - --path ${MODEL_DIR}/checkpoint.pt \ - --task translation_multi_simple_epoch \ - --langs "en,ha,is,ja,cs,ru,zh,de" \ - --lang-pairs $SRC-$TGT \ - --bpe "sentencepiece" \ - --sentencepiece-model ${MODEL_DIR}/sentencepiece.model \ - --buffer-size 1024 \ - --batch-size 10 -s $SRC -t $TGT \ - --decoder-langtok \ - --encoder-langtok src \ - --beam 5 \ - --lenpen 1.0 \ - --fp16 > $TMP_DIR/${SRC}-${TGT}.gen_log - -cat $TMP_DIR/$SRC-$TGT.gen_log | grep -P "^D-" | cut -f3 > $TMP_DIR/$SRC-$TGT.hyp - -# Calculate BLEU score -sacrebleu -l $SRC-$TGT $REF_FILE < $TMP_DIR/$SRC-$TGT.hyp diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wmt21/scripts/normalize-punctuation.perl b/audio/safeear/safeear_repo/fairseq_ours/examples/wmt21/scripts/normalize-punctuation.perl deleted file mode 100644 index a7c0750f5844ab3a10a497714ce55dcf8680cf49..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wmt21/scripts/normalize-punctuation.perl +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env perl -# -# This file is part of moses. Its use is licensed under the GNU Lesser General -# Public License version 2.1 or, at your option, any later version. - -use warnings; -use strict; - -my $language = "en"; -my $PENN = 0; - -while (@ARGV) { - $_ = shift; - /^-b$/ && ($| = 1, next); # not buffered (flush each line) - /^-l$/ && ($language = shift, next); - /^[^\-]/ && ($language = $_, next); - /^-penn$/ && ($PENN = 1, next); -} - -while(<STDIN>) { - s/\r//g; - # remove extra spaces - s/\(/ \(/g; - s/\)/\) /g; s/ +/ /g; - s/\) ([\.\!\:\?\;\,])/\)$1/g; - s/\( /\(/g; - s/ \)/\)/g; - s/(\d) \%/$1\%/g; - s/ :/:/g; - s/ ;/;/g; - # normalize unicode punctuation - if ($PENN == 0) { - s/\`/\'/g; - s/\'\'/ \" /g; - } - - s/„/\"/g; - s/“/\"/g; - s/”/\"/g; - s/–/-/g; - s/—/ - /g; s/ +/ /g; - s/´/\'/g; - s/([a-z])‘([a-z])/$1\'$2/gi; - s/([a-z])’([a-z])/$1\'$2/gi; - s/‘/\'/g; - s/‚/\'/g; - s/’/\"/g; - s/''/\"/g; - s/´´/\"/g; - s/…/.../g; - # French quotes - s/ « / \"/g; - s/« /\"/g; - s/«/\"/g; - s/ » /\" /g; - s/ »/\"/g; - s/»/\"/g; - # handle pseudo-spaces - s/ \%/\%/g; - s/nº /nº /g; - s/ :/:/g; - s/ ºC/ ºC/g; - s/ cm/ cm/g; - s/ \?/\?/g; - s/ \!/\!/g; - s/ ;/;/g; - s/, /, /g; s/ +/ /g; - - # English "quotation," followed by comma, style - if ($language eq "en") { - s/\"([,\.]+)/$1\"/g; - } - # Czech is confused - elsif ($language eq "cs" || $language eq "cz") { - } - # German/Spanish/French "quotation", followed by comma, style - else { - s/,\"/\",/g; - s/(\.+)\"(\s*[^<])/\"$1$2/g; # don't fix period at end of sentence - } - - - if ($language eq "de" || $language eq "es" || $language eq "cz" || $language eq "cs" || $language eq "fr") { - s/(\d) (\d)/$1,$2/g; - } - else { - s/(\d) (\d)/$1.$2/g; - } - print $_; -} diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/wmt21/scripts/replace-unicode-punctuation.perl b/audio/safeear/safeear_repo/fairseq_ours/examples/wmt21/scripts/replace-unicode-punctuation.perl deleted file mode 100644 index faed2cd9d86af7ad96300571b3a7b1037cc21e79..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/wmt21/scripts/replace-unicode-punctuation.perl +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env perl -# -# This file is part of moses. Its use is licensed under the GNU Lesser General -# Public License version 2.1 or, at your option, any later version. - -use warnings; -use strict; - -while (@ARGV) { - $_ = shift; - /^-b$/ && ($| = 1, next); # not buffered (flush each line) -} - -#binmode(STDIN, ":utf8"); -#binmode(STDOUT, ":utf8"); - -while(<STDIN>) { - s/，/,/g; - s/。 */. /g; - s/、/,/g; - s/”/"/g; - s/“/"/g; - s/∶/:/g; - s/：/:/g; - s/？/\?/g; - s/《/"/g; - s/》/"/g; - s/）/\)/g; - s/！/\!/g; - s/（/\(/g; - s/；/;/g; - s/１/1/g; - s/」/"/g; - s/「/"/g; - s/０/0/g; - s/３/3/g; - s/２/2/g; - s/５/5/g; - s/６/6/g; - s/９/9/g; - s/７/7/g; - s/８/8/g; - s/４/4/g; - s/． */. /g; - s/～/\~/g; - s/’/\'/g; - s/…/\.\.\./g; - s/━/\-/g; - s/〈/\</g; - s/〉/\>/g; - s/【/\[/g; - s/】/\]/g; - s/％/\%/g; - print $_; -} diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/womens_bios/README.md b/audio/safeear/safeear_repo/fairseq_ours/examples/womens_bios/README.md deleted file mode 100644 index 07d06468870c5d124f44f87d9c27158b6210cd58..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/womens_bios/README.md +++ /dev/null @@ -1,81 +0,0 @@ -# Wikipedia Biographies of Women - - -## Training: - -The training dataset is created based on WikiSum, a dataset created from the paper [Generating Wikipedia by Summarizing Long Sequences](https://arxiv.org/pdf/1801.10198.pdf). The dataset needs to be generated following the instructions in this [Github Repository](https://github.com/tensorflow/tensor2tensor/tree/master/tensor2tensor/data_generators/wikisum). - -### How is the WikiSum dataset structured? - -Overall, the task in WikiSum was to generate the entire Wikipedia article based on the contents of the top 10 Google Search Results. The authors provide a way for people to recreate their work. In the WikiSum Github, there are two options for the dataset recreation --- the first is to use CommonCrawl (a static, open source crawl of the web) and the second to do Live Web Fetches. The second has higher coverage, but the content is subject to change and difficult to fetch. We used the static, Commoncrawl version. This can be downloaded following the Github repo instructions, though note it will require usage of Google Cloud. - -Note: in our experience, it also requires requesting that the resource limit of the Google Cloud instance be raised, which requires emailing. - -Note: Having higher coverage in the training dataset would be expected to improve the model quality. There are many instances in the dataset where the training input (web evidence) does not contain sufficient content for producing the desired Wikipedia article. This may harm the model's ability to learn to retrieve, look at the input evidence, and overall could contribute to increased challenges in generating verifiable Wikipedia biographies. - -### How do you go from WikiSum dataset to Biography dataset? - -The WikiSum dataset is for Wikipedia in general, not just biographies. We do this by querying WikiData to see if the Wikipedia article has an occupation, with the thought that all articles with occupations are probably biographies. - - -## Evaluation: - -You can download the dataset and baseline model with the following command: - -``` -wget -N 'https://dl.fbaipublicfiles.com/fairseq/womenbios_dataset.zip' -wget -N 'https://dl.fbaipublicfiles.com/fairseq/gpt2_bpe/encoder.json' -wget -N 'https://dl.fbaipublicfiles.com/fairseq/gpt2_bpe/vocab.bpe' -wget -N 'https://dl.fbaipublicfiles.com/fairseq/gpt2_bpe/dict.txt' -``` - -We provide the full text Wikipedia articles split into four categories: -- Women in Africa -- Women in Asia -- Women in Science -- Women -We note that these are not exhaustive intersectional categories and mainly stem from personal interest. - -We also provide the URL of the Wikipedia article. Note that Wikipedia articles are constantly being improved, edited, and changed. Thus, it's completely possible that the Wikipedia article on Wikipedia has been lovingly improved by other Wikipedia editors. - -To get the occupations of each biographical subject, we use WikiData. We provide a sample script to do this. We also provide the raw output of this query. - -The final part of the evaluation dataset is to query web evidence for each of the biographical subjects. This is the part of the evaluation dataset that requires the most improvement. As we discuss in our paper, one of the major reasons why it is difficult to write biographies for sometimes very well qualified women is that there is not information online about them. Further, the search engine may not find it. We encourage others to improve upon this part of the data, as even re-querying again on the internet may find new, updated sources of information as the web is constantly evolving. - -We use the search engine from [Internet-Augmented Dialogue Generation](https://arxiv.org/abs/2107.07566), see [project URL](https://parl.ai/projects/sea/) to do the search queries. Note: we remove wikipedia site sources from our query (or we'd query the data itself). However, it's possible Wikipedia information can be copied around in multiple forms on the web, linked with edits, etc. - - -## Section by Section Generation: - -Wikipedia articles are split into sections, which are usually separated by headings. These headings can be separated in the article text by looking for these equal signs (==), where the number of equal signs usually signals if you are looking at a toplevel heading or a subheading, etc. An example regex that you can use is: - -` -section_header_re = re.compile(r"(?<!=)==([^=]+)==(?!=)") -` - - -## List of Notes: -- People can have multiple occupations, and we keep all occupations that we query from WikiData - - -## List of Possible Improvement Areas: -Using a larger generative pre-trained model, larger-scale retrieval, a retrieval encoder specialized to Wikipedia (or biographies), tuning all of the training & generation parameters exhaustively --- and the like --- would most likely be very useful. Overall, we hope that this is a starting point for others who might be interested in focusing on how we can help address the gender gap on Wikipedia. - - -## Interested in Wikipedia and Gender Gap? -You might want to check out: -- https://humaniki.wmcloud.org/ -- https://en.wikipedia.org/wiki/Wikipedia:WikiProject_Women_in_Red and https://wikimediafoundation.org/news/2018/10/18/women-in-red-wikiproject/ -- https://meta.wikimedia.org/wiki/Whose_Knowledge%3F/VisibleWikiWomen -- https://www.ted.com/talks/jess_wade_a_voice_for_diversity_in_science - -and thanks again to all of the Wikipedia editors and the entire community that is already working so hard to write amazing articles for diverse groups of people. - - -# LICENSE -This is licensed under CC-BY-NC, however portions of the dataset are available under separate license terms: text sourced from Wikipedia is licensed under CC-BY-SA. - - - - - diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/womens_bios/query_occupations_from_wikidata.py b/audio/safeear/safeear_repo/fairseq_ours/examples/womens_bios/query_occupations_from_wikidata.py deleted file mode 100644 index af27e98cbbf2e5ab5adc46e21ef3e41ebe619049..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/womens_bios/query_occupations_from_wikidata.py +++ /dev/null @@ -1,35 +0,0 @@ -import sys - -from SPARQLWrapper import JSON, SPARQLWrapper - -endpoint_url = "https://query.wikidata.org/sparql" - -with open("/your/urls/here") as f: - data = f.readlines() -urls = [i.strip() for i in data] - -def get_results(endpoint_url, URL): - query = f"""SELECT ?uriLabel ?occupation ?occupationLabel ?dob ?dobLabel WHERE {{ - <{URL}> schema:about ?uri . - ?uri wdt:P106 ?occupation . - SERVICE wikibase:label {{ bd:serviceParam wikibase:language "en" }} - }}""" - user_agent = "WDQS-example Python/%s.%s" % (sys.version_info[0], sys.version_info[1]) - sparql = SPARQLWrapper(endpoint_url, agent=user_agent) - sparql.setQuery(query) - sparql.setReturnFormat(JSON) - return sparql.query().convert() - -all_occupations = [] -for URL in urls: - results = get_results(endpoint_url, URL) - occupations = [] - for result in results["results"]["bindings"]: - occupations.append(result['occupationLabel']['value']) - all_occupations.append(result['uriLabel']['value'] + ", " + ", ".join(occupations)) - -assert(len(all_occupations) == len(urls)) - -with open("/your/file/output/here", "w") as o: - for line in all_occupations: - o.write(line.strip() + "\n") \ No newline at end of file diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/xformers/README.md b/audio/safeear/safeear_repo/fairseq_ours/examples/xformers/README.md deleted file mode 100644 index 400a74d5366b7cbd0eab63af1901a45346fa3de8..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/xformers/README.md +++ /dev/null @@ -1,43 +0,0 @@ -# Using xFormers with FairSeq - -[xFormers](https://github.com/facebookresearch/xformers) is a xFormers is a modular library for flexibly generating transformer architectures with interoperable and optimized building blocks. -The current integration allows for FairSeq users to use an attention variant available in the xFormers repository. - -In order to enable xFormers, all that needs to be passed in is a string representing an [xFormers attention config](https://github.com/facebookresearch/xformers/blob/5f754129bfb1ea53747b1ab2077261ea762faa47/xformers/components/attention/base.py#L18). - -The various attention variants can be found [here](https://github.com/facebookresearch/xformers/tree/main/xformers/components/attention). -These include sparse attention and blocksparse attention. - -For example, you could pass in the following args: - ```python -decoder_xformers_att_config = '{"name": "scaled_dot_product"}' - -encoder_xformers_att_config = '{"name": "linformer", "seq_len": "256"}' - ``` - -In order to use blocksparse attention you would have to additionally pass in a blocksparse layout and blocksize. For example: - - ```python - - xformers_att_config = '{"name": "scaled_dot_product"}' - xformers_blocksparse_blocksize = 16 - xformers_blocksparse_layout = torch.ones( - seq_len // xformers_blocksparse_blocksize, - seq_len // xformers_blocksparse_blocksize, - ) - - xf_blocksparse_mha = ( - MultiheadAttention( - embedding, - num_heads, - dropout=0.0, - add_zero_attn=add_zero_attn, - xformers_att_config=xformers_att_config, - xformers_blocksparse_layout=xformers_blocksparse_layout, - xformers_blocksparse_blocksize=xformers_blocksparse_blocksize, - ) - - ``` - -The xFormers repository currenlty has benchmarks on the [runtime](https://github.com/facebookresearch/xformers/blob/main/docs/plots/runtime_vs_attention.png) -and [memory usage](https://github.com/facebookresearch/xformers/blob/main/docs/plots/memory_vs_attention.png) of the various attentions. diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/xglm/README.md b/audio/safeear/safeear_repo/fairseq_ours/examples/xglm/README.md deleted file mode 100644 index fb630f6f07a6039d8c369762663760a4fb6ffe75..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/xglm/README.md +++ /dev/null @@ -1,181 +0,0 @@ -# Few-shot Learning with Multilingual Language Models - -## Introduction - -In this work, we train a family of multilingual generative language models, dubbed XGLM, on a balanced corpus covering a diverse set of languages, and study their few- and zero-shot learning capabilities in a wide range of tasks. Our largest model with 7.5 billion parameters sets new state of the art in few-shot learning on more than 20 representative languages, outperforming GPT-3 of comparable size in multilingual commonsense reasoning (+7.4 accuracy points for 0-shot, +9.4 for 4-shot) and natural language inference (+5.4 for 0-shot, +5.4 for 4-shot). We have included a [model card](model_card.md) of XGLM for transparency and accountability. - -## Data and Languages -XGLM models are trained on a new multilingual corpus extracted from CommonCrawl (CC100-XL), a significantly larger multilingual dataset covering 68 Common Crawl (CC) snapshots (from [Summer 2013](http://commoncrawl.org/2013/11/new-crawl-data-available/) to [March/April 2020](https://commoncrawl.org/2020/04/march-april-2020-crawl-archive-now-available/) consisting of 134 languages. The detailed languages and data statistics are reported in the paper (Table A.1). - -## Pre-trained models - -Model | Layers | Model Dim | FFN Dim | Languages | Download ----|---|---|---|---|--- -`XGLM 564M` | 24 | 1024 | 4096 | trained on 30 languages| [xglm.564M.tar.gz](https://dl.fbaipublicfiles.com/fairseq/models/xglm/xglm.564M.tar.gz) -`XGLM 1.7B` | 24 | 2048 | 8192 | trained on 30 languages| [xglm.1.7B.tar.gz](https://dl.fbaipublicfiles.com/fairseq/models/xglm/xglm.1.7B.tar.gz) -`XGLM 2.9B` | 48 | 2048 | 8192 | trained on 30 languages| [xglm.2.9B.tar.gz](https://dl.fbaipublicfiles.com/fairseq/models/xglm/xglm.2.9B.tar.gz) -`XGLM 7.5B` | 32 | 4096 | 16384 | trained on 30 languages| [xglm.7.5B.tar.gz](https://dl.fbaipublicfiles.com/fairseq/models/xglm/xglm.7.5B.tar.gz) -`XGLM 4.5B` | 48 | 2048 | 16384 | trained on 134 languages| [xglm.4.5B.tar.gz](https://dl.fbaipublicfiles.com/fairseq/models/xglm/xglm.4.5B.tar.gz) - -## Pre-training Data Format -Our models were pre-trained with data in the following format (i.e. paragraphs are separated with new lines and documents were separated with double new lines). -``` -<doc0,para0,tok0> ... <doc0,para0,tokX0> # X0: number of tokens in para0 of doc0 -<doc0,para1,tok0> ... <doc0,para1,tokY0> # Y0: number of tokens in para1 of doc0 - -<doc1,para0,tok0> ... <doc1,para0,tokX1> # X1: number of tokens in para0 of doc1 -<doc1,para1,tok0> ... <doc1,para1,tokY1> # Y1: number of tokens in para1 of doc1 - -... -``` -Fairseq's preprocessing replaces newlines with the end-of-sentence symbol (`</s>`). As a result, the models never saw newline characters during pretraining and the same preprocessing should be run prior to few-shot inference to maximize performance. For example, our language model scoring function has `replace_newlines_with_eos` argument to trigger this preprocessing: -```python -from fairseq.models.transformer_lm import TransformerLanguageModel - -model_dir = 'path_to_decompressed_tar_gz_dir' -lm = TransformerLanguageModel.from_pretrained(model_dir, bpe='sentencepiece') - -text = """First paragraph of the first document. -Second paragraph of the first document. - -First paragraph of the second document. -""" -tokens = lm.score(text, replace_newlines_with_eos=True)['tokens'] -assert '\n' not in lm.decode(tokens) # no newlines were encoded -``` - -## Evaluation - -### Example (COPA) - -The following snippet show how to evaluate our models on the Choice of Plausible Alternatives (COPA) task, using examples in English, Chinese and Hindi. - -```python -data_samples = { - 'en': [ - { - "premise": "I wanted to conserve energy.", - "choice1": "I swept the floor in the unoccupied room.", - "choice2": "I shut off the light in the unoccupied room.", - "question": "effect", - "label": "1" - }, - { - "premise": "The flame on the candle went out.", - "choice1": "I blew on the wick.", - "choice2": "I put a match to the wick.", - "question": "cause", - "label": "0" - } - ], - 'zh': [ - { - "premise": "我想节约能源。", - "choice1": "我在空着的房间里扫了地板。", - "choice2": "我把空房间里的灯关了。", - "question": "effect", - "label": "1" - }, - { - "premise": "蜡烛上的火焰熄灭了。", - "choice1": "我吹灭了灯芯。", - "choice2": "我把一根火柴放在灯芯上。", - "question": "cause", - "label": "0" - } - ], - 'hi': [ - { - "premise": "M te vle konsève enèji.", - "choice1": "Mwen te fin baleye chanm lib la.", - "choice2": "Mwen te femen limyè nan chanm lib la.", - "question": "effect", - "label": "1" - }, - { - "premise": "Flam bouji a te etenn.", - "choice1": "Mwen te soufle bouji a.", - "choice2": "Mwen te limen mèch bouji a.", - "question": "cause", - "label": "0" - } - ] -} -``` -In this example, we format the examples use the non-verbal prompts `{premise}\n{choice1}` and `{premise}\n{choice2}`, which are shared by all three languages. -```python -from fairseq.models.transformer_lm import TransformerLanguageModel - -model_dir = 'path_to_decompressed_tar_gz_dir' -lm = TransformerLanguageModel.from_pretrained(model_dir, bpe='sentencepiece') -lm = lm.eval() -lm = lm.half() -lm = lm.cuda() - -def get_logprobs(prompt): - import re - prompt = re.sub('\n+' , '\n', prompt) # collapse repeated newlines, which indicate separate documents - return lm.score(prompt, replace_newlines_with_eos=True)['positional_scores'] - -# Zero-shot evaluation for the Choice of Plausible Alternatives (COPA) task. -# A return value of 0 indicates that the first alternative is more plausible, -# while 1 indicates that the second alternative is more plausible. -def COPA_eval(prompt, alternative1, alternative2): - lprob1 = get_logprobs(prompt + "\n" + alternative1).sum() - lprob2 = get_logprobs(prompt + "\n" + alternative2).sum() - return 0 if lprob1 > lprob2 else 1 - -for lang in ['en', 'zh', 'hi']: - for idx, example in enumerate(data_samples[lang]): - predict = COPA_eval(example["premise"], example["choice1"], example["choice2"]) - print(f'{lang}-{idx}', predict, example['label']) - -# en-0 1 1 -# en-1 0 0 -# zh-0 1 1 -# zh-1 0 0 -# hi-0 1 1 -# hi-1 0 0 -``` - -## Preprint -[Few-shot Learning with Multilingual Language Models](https://arxiv.org/abs/2112.10668). -Xi Victoria Lin*, Todor Mihaylov, Mikel Artetxe, Tianlu Wang, Shuohui Chen, Daniel Simig, Myle Ott, Naman Goyal, Shruti Bhosale, Jingfei Du, Ramakanth Pasunuru, Sam Shleifer, Punit Singh Koura, Vishrav Chaudhary, Brian O'Horo, Jeff Wang, Luke Zettlemoyer, Zornitsa Kozareva, Mona Diab, Veselin Stoyanov, Xian Li* (* Equal Contribution). -ArXiv 2021. - -## Citation -``` -@article{DBLP:journals/corr/abs-2112-10668, - author = {Xi Victoria Lin and - Todor Mihaylov and - Mikel Artetxe and - Tianlu Wang and - Shuohui Chen and - Daniel Simig and - Myle Ott and - Naman Goyal and - Shruti Bhosale and - Jingfei Du and - Ramakanth Pasunuru and - Sam Shleifer and - Punit Singh Koura and - Vishrav Chaudhary and - Brian O'Horo and - Jeff Wang and - Luke Zettlemoyer and - Zornitsa Kozareva and - Mona T. Diab and - Veselin Stoyanov and - Xian Li}, - title = {Few-shot Learning with Multilingual Language Models}, - journal = {CoRR}, - volume = {abs/2112.10668}, - year = {2021}, - url = {https://arxiv.org/abs/2112.10668}, - eprinttype = {arXiv}, - eprint = {2112.10668}, - timestamp = {Tue, 04 Jan 2022 15:59:27 +0100}, - biburl = {https://dblp.org/rec/journals/corr/abs-2112-10668.bib}, - bibsource = {dblp computer science bibliography, https://dblp.org} -} -``` diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/xglm/model_card.md b/audio/safeear/safeear_repo/fairseq_ours/examples/xglm/model_card.md deleted file mode 100644 index 0cdd20fbe0cee5e3ab77f4682ec5fe5aae3ef365..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/xglm/model_card.md +++ /dev/null @@ -1,152 +0,0 @@ -# XGLM multilingual model -## Version 1.0.0 - -### Model developer -Meta AI - -### Model type -A family of multilingual autoregressive language models (ranging from 564 million to 7.5 billion parameters) trained on a balanced corpus of a diverse set of languages. The language model can learn tasks from natural language descriptions and a few examples. - -### Model Feedback Channel -https://github.com/pytorch/fairseq - -## Intended use -### Primary intended use -For research purposes only, e.g. reproducing model evaluation results. Generation is only used in a limited capacity for explanation/justification or for prompting/probing/priming for class labels. - -### Out of scope uses -The primary purpose of the model is not to generate language, although the model is capable of doing that. - -## Potential risks -This section lists the potential risks associated with using the model. - -### Relevant factors -Based on known problems with NLP technology, potential relevant factors include output correctness, robustness, bias (gender, profession, race and religion), etc. - -### Evaluation factors -The model was evaluated on hate speech detection and occupation identification. -* Hate speech detection (Huang et al. (2020)) - A safety task to test language models’ ability to identify hateful and offensive text. -* Occupation identification (De-Arteaga et al., 2019), (Zhao et al., 2020) - A bias task to study language models’ performance divergence between different gender groups on the task of occupation identification. - -## Metrics -### Model performance measures -The XGLM model was primarily evaluated on -1. Zero shot and few shot learning by looking at per-language performance on tasks spanning commonsense reasoning (XCOPA, XWinograd), natural language inference (XNLI) and paraphrasing (PAWS-X). The model is also evaluated on XStoryCloze, a new dataset created by Meta AI. -2. Cross lingual transfer through templates and few-shot examples. -3. Knowledge probing - Evaluate to what extent the XGLM model can effectively store factual knowledge in different languages using the mLAMA benchmark. -4. Translation - We report machine translation results on WMT benchmarks and a subset of FLORES-101 in the main paper. - -The model was also evaluated on hate speech datasets introduced by Huang et al. (2020) and an occupation identification dataset by De-Arteaga et al. 2019 to identify bias in the model. - -### Approaches to handle uncertainty -Report confidence intervals, variance metrics for the model performance metrics. Few-shot evaluation was conducted with different sampling with 5 seeds. We reported statistical significance. - -## Evaluation data -## Zero Shot and Few Shot evaluation - -### XNLI (Conneau et al., 2018) -#### Description -The Cross-lingual Natural Language Inference (XNLI) corpus is the extension of the Multi-Genre NLI (MultiNLI) corpus to 15 languages. The dataset was created by manually translating the validation and test sets of MultiNLI into each of those 15 languages. - -### XStoryCloze -#### Description -A new dataset created by Meta AI by translating the validation split of the English StoryCloze dataset (Mostafazadeh et al., 2016) (Spring 2016 version) to 10 other typologically diverse languages (ru, zh Simplified, es Latin America, ar, hi, id, te, sw, eu, my). - -### XCOPA (Ponti et al., 2020) -#### Description -The Cross-lingual Choice of Plausible Alternatives (XCOPA) dataset is a benchmark to evaluate the ability of machine learning models to transfer commonsense reasoning across languages. The dataset is the translation and reannotation of the English COPA (Roemmele et al. 2011) and covers 11 languages from 11 families and several areas around the globe. - -### XWinograd (Tikhonov and Ryabinin, 2021) -#### Description -XWinograd is a multilingual collection of Winograd Schemas in six languages that can be used for evaluation of cross-lingual commonsense reasoning capabilities. - -### PAWS-X (Yang et al., 2019) -#### Description -PAWS-X contains 23,659 human translated PAWS evaluation pairs and 296,406 machine translated training pairs in six typologically distinct languages: French, Spanish, German, Chinese, Japanese, and Korean. All translated pairs are sourced from examples in PAWS-Wiki. - -## Responsible AI (RAI) evaluation -### Hate speech (Huang et al. 2020) -This is a multilingual Twitter corpus for the task of hate speech detection with inferred four author demographic factors: age, country, gender and race/ethnicity. The corpus covers five languages: English, Italian, Polish, Portuguese and Spanish. - -### Bias dataset (De-Arteaga et al. 2019) -The aim of this dataset is to study the gender bias of models that identify a person’s occupation from their bios. - ----- - -## Training data -### CC100-XL -#### Description -Following the recent success of multilingual self-supervised pre-training (Devlin et al., 2019; Lample and Conneau, 2019; Con; Xue et al., 2020; Goyal et al., 2021a; Liu et al., 2020), we train our language models on a mixture of monolingual text of different languages. We extended the pipeline used for mining the CC100 corpus to generate CC100-XL, a significantly larger multilingual dataset covering 68 Common Crawl snapshots (from Summer 2013 to March/April 2020) and 134 languages. - -More details on the CC100-XL dataset can be found in the Appendix section of the paper. - -## RAI Dimensions -### Fairness (Bias and inclusion) -The XGLM model was evaluated on Hate speech and bias identification datasets. For hate speech, we observe that across the 5 languages in the dataset, in context learning results are only slightly better than random (50%). Another interesting observation is that most few shot results are worse than zero-shot, which indicates that the model is not able to utilize examples using the templates described in the paper. For bias identification, the XGLM (6.7B) English only model achieves the best performance on English and Spanish, while the GPT-3 model of comparable size (6.7B) model achieves the best in French. On certain occupations (e.g. model and teacher), XGLM 6.7B En only model and GPT-3 (6.7B) have very significant bias while XGLM 7.5B is much less biased. - -### Privacy and security -The XGLM model did not have any special Privacy and Security considerations. The training data and evaluation data were both public and went through standard Meta AI Privacy and licensing procedures. - -### Transparency and control -In the spirit of transparency and accountability we have created this model card and a data card for the CC100-XL which can be found in the Appendix section of the paper. - -### Efficiency (Green AI) -From an engineering perspective, XGLM pertains to a family of models that represent single unified models catering to many languages which have wide application across many applications. Such a unified single model saves on carbon footprint as well as energy consumption (comparing to the alternative: separate models for different languages) leading to more energy efficiency. A single model, despite having the risk of being a single point of failure, has the powerful incentive of being easier to maintain, access, distribute, and track. - -## References -Edoardo Maria Ponti, Goran Glavas, Olga Majewska, Qianchu Liu, Ivan Vulic, and Anna Korhonen. 2020. XCOPA: A multilingual dataset for causal commonsense reasoning. In Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing, EMNLP 2020, Online, November 16-20, 2020, pages 2362–2376. Association for Computational Linguistics. -XCOPA Dataset | Papers With Code - -Alexey Tikhonov and Max Ryabinin. 2021. It’s all in the heads: Using attention heads as a baseline for cross-lingual transfer in commonsense reasoning. In Findings of the Association for Computational Linguistics: ACL/IJCNLP 2021, Online Event, August 1-6, 2021, volume ACL/IJCNLP 2021 of Findings of ACL, pages 3534–3546. Association for Computational Linguistics. -XWINO Dataset | Papers With Code (XWinograd) - -Yinfei Yang, Yuan Zhang, Chris Tar, and Jason Baldridge. 2019. PAWS-X: A cross-lingual adversarial dataset for paraphrase identification. CoRR, abs/1908.11828. -PAWS-X Dataset | Papers With Code - -Alexis Conneau, Guillaume Lample, Ruty Rinott, Adina Williams, Samuel R. Bowman, Holger Schwenk, and Veselin Stoyanov. 2018. XNLI: evaluating cross-lingual sentence representations. CoRR, abs/1809.05053. -XNLI Dataset | Papers With Code - -Xiaolei Huang, Linzi Xing, Franck Dernoncourt, and Michael Paul. 2020. Multilingual twitter corpus and baselines for evaluating demographic bias in hate speech recognition. In Proceedings of the 12th Language Resources and Evaluation Conference, pages 1440–1448. - -Maria De-Arteaga, Alexey Romanov, Hanna Wallach, Jennifer Chayes, Christian Borgs, Alexandra Chouldechova, Sahin Geyik, Krishnaram Kenthapadi, and Adam Tauman Kalai. 2019. Bias in bios: A case study of semantic representation bias in a high-stakes setting. In proceedings of the Conference on Fairness, Accountability, and Transparency, pages 120–128. - -Nasrin Mostafazadeh, Nathanael Chambers, Xiaodong He, Devi Parikh, Dhruv Batra, Lucy Vanderwende, Pushmeet Kohli, James F. Allen. A Corpus and Evaluation Framework for Deeper Understanding of Commonsense Stories. CoRR abs/1604.01696. - -Jieyu Zhao, Subhabrata Mukherjee, Saghar Hosseini, Kai-Wei Chang, and Ahmed Hassan Awadallah. 2020. Gender bias in multilingual embeddings and crosslingual transfer. In Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics, pages 2896–2907. - -## Citation details -``` -@article{DBLP:journals/corr/abs-2112-10668, - author = {Xi Victoria Lin and - Todor Mihaylov and - Mikel Artetxe and - Tianlu Wang and - Shuohui Chen and - Daniel Simig and - Myle Ott and - Naman Goyal and - Shruti Bhosale and - Jingfei Du and - Ramakanth Pasunuru and - Sam Shleifer and - Punit Singh Koura and - Vishrav Chaudhary and - Brian O'Horo and - Jeff Wang and - Luke Zettlemoyer and - Zornitsa Kozareva and - Mona T. Diab and - Veselin Stoyanov and - Xian Li}, - title = {Few-shot Learning with Multilingual Language Models}, - journal = {CoRR}, - volume = {abs/2112.10668}, - year = {2021}, - url = {https://arxiv.org/abs/2112.10668}, - eprinttype = {arXiv}, - eprint = {2112.10668}, - timestamp = {Tue, 04 Jan 2022 15:59:27 +0100}, - biburl = {https://dblp.org/rec/journals/corr/abs-2112-10668.bib}, - bibsource = {dblp computer science bibliography, https://dblp.org} -} -``` diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/xlmr/README.md b/audio/safeear/safeear_repo/fairseq_ours/examples/xlmr/README.md deleted file mode 100644 index bba7910e303e5bc18b570b8a3877db59e1762d43..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/xlmr/README.md +++ /dev/null @@ -1,144 +0,0 @@ -# Unsupervised Cross-lingual Representation Learning at Scale (XLM-RoBERTa) -https://arxiv.org/pdf/1911.02116.pdf - -# Larger-Scale Transformers for Multilingual Masked Language Modeling -https://arxiv.org/pdf/2105.00572.pdf - - -## What's New: -- June 2021: `XLMR-XL` AND `XLMR-XXL` models released. - -## Introduction - -`XLM-R` (`XLM-RoBERTa`) is a generic cross lingual sentence encoder that obtains state-of-the-art results on many cross-lingual understanding (XLU) benchmarks. It is trained on `2.5T` of filtered CommonCrawl data in 100 languages (list below). - - Language | Language|Language |Language | Language ----|---|---|---|--- -Afrikaans | Albanian | Amharic | Arabic | Armenian -Assamese | Azerbaijani | Basque | Belarusian | Bengali -Bengali Romanize | Bosnian | Breton | Bulgarian | Burmese -Burmese zawgyi font | Catalan | Chinese (Simplified) | Chinese (Traditional) | Croatian -Czech | Danish | Dutch | English | Esperanto -Estonian | Filipino | Finnish | French | Galician -Georgian | German | Greek | Gujarati | Hausa -Hebrew | Hindi | Hindi Romanize | Hungarian | Icelandic -Indonesian | Irish | Italian | Japanese | Javanese -Kannada | Kazakh | Khmer | Korean | Kurdish (Kurmanji) -Kyrgyz | Lao | Latin | Latvian | Lithuanian -Macedonian | Malagasy | Malay | Malayalam | Marathi -Mongolian | Nepali | Norwegian | Oriya | Oromo -Pashto | Persian | Polish | Portuguese | Punjabi -Romanian | Russian | Sanskrit | Scottish Gaelic | Serbian -Sindhi | Sinhala | Slovak | Slovenian | Somali -Spanish | Sundanese | Swahili | Swedish | Tamil -Tamil Romanize | Telugu | Telugu Romanize | Thai | Turkish -Ukrainian | Urdu | Urdu Romanize | Uyghur | Uzbek -Vietnamese | Welsh | Western Frisian | Xhosa | Yiddish - -## Pre-trained models - -Model | Description | #params | vocab size | Download ----|---|---|---|--- -`xlmr.base` | XLM-R using the BERT-base architecture | 250M | 250k | [xlm.base.tar.gz](https://dl.fbaipublicfiles.com/fairseq/models/xlmr.base.tar.gz) -`xlmr.large` | XLM-R using the BERT-large architecture | 560M | 250k | [xlm.large.tar.gz](https://dl.fbaipublicfiles.com/fairseq/models/xlmr.large.tar.gz) -`xlmr.xl` | XLM-R (`layers=36, model_dim=2560`) | 3.5B | 250k | [xlm.xl.tar.gz](https://dl.fbaipublicfiles.com/fairseq/models/xlmr/xlmr.xl.tar.gz) -`xlmr.xxl` | XLM-R (`layers=48, model_dim=4096`) | 10.7B | 250k | [xlm.xxl.tar.gz](https://dl.fbaipublicfiles.com/fairseq/models/xlmr/xlmr.xxl.tar.gz) - -## Results - -**[XNLI (Conneau et al., 2018)](https://arxiv.org/abs/1809.05053)** - -Model | average | en | fr | es | de | el | bg | ru | tr | ar | vi | th | zh | hi | sw | ur ----|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|--- -`roberta.large.mnli` _(TRANSLATE-TEST)_ | 77.8 | 91.3 | 82.9 | 84.3 | 81.2 | 81.7 | 83.1 | 78.3 | 76.8 | 76.6 | 74.2 | 74.1 | 77.5 | 70.9 | 66.7 | 66.8 -`xlmr.large` _(TRANSLATE-TRAIN-ALL)_ | 83.6 | 89.1 | 85.1 | 86.6 | 85.7 | 85.3 | 85.9 | 83.5 | 83.2 | 83.1 | 83.7 | 81.5 | 83.7 | 81.6 | 78.0 | 78.1 -`xlmr.xl` _(TRANSLATE-TRAIN-ALL)_ | 85.4 | 91.1 | 87.2 | 88.1 | 87.0 | 87.4 | 87.8 | 85.3 | 85.2 | 85.3 | 86.2 | 83.8 | 85.3 | 83.1 | 79.8 | 78.2 | 85.4 -`xlmr.xxl` _(TRANSLATE-TRAIN-ALL)_ | 86.0 | 91.5 | 87.6 | 88.7 | 87.8 | 87.4 | 88.2 | 85.6 | 85.1 | 85.8 | 86.3 | 83.9 | 85.6 | 84.6 | 81.7 | 80.6 - -**[MLQA (Lewis et al., 2018)](https://arxiv.org/abs/1910.07475)** - -Model | average | en | es | de | ar | hi | vi | zh ----|---|---|---|---|---|---|---|--- -`BERT-large` | - | 80.2/67.4 | - | - | - | - | - | - -`mBERT` | 57.7 / 41.6 | 77.7 / 65.2 | 64.3 / 46.6 | 57.9 / 44.3 | 45.7 / 29.8| 43.8 / 29.7 | 57.1 / 38.6 | 57.5 / 37.3 -`xlmr.large` | 70.7 / 52.7 | 80.6 / 67.8 | 74.1 / 56.0 | 68.5 / 53.6 | 63.1 / 43.5 | 69.2 / 51.6 | 71.3 / 50.9 | 68.0 / 45.4 -`xlmr.xl` | 73.4 / 55.3 | 85.1 / 72.6 | 66.7 / 46.2 | 70.5 / 55.5 | 74.3 / 56.9 | 72.2 / 54.7 | 74.4 / 52.9 | 70.9 / 48.5 -`xlmr.xxl` | 74.8 / 56.6 | 85.5 / 72.4 | 68.6 / 48.4 | 72.7 / 57.8 | 75.4 / 57.6 | 73.7 / 55.8 | 76.0 / 55.0 | 71.7 / 48.9 - - -## Example usage - -##### Load XLM-R from torch.hub (PyTorch >= 1.1): -```python -import torch -xlmr = torch.hub.load('pytorch/fairseq:main', 'xlmr.large') -xlmr.eval() # disable dropout (or leave in train mode to finetune) -``` - -##### Load XLM-R (for PyTorch 1.0 or custom models): -```python -# Download xlmr.large model -wget https://dl.fbaipublicfiles.com/fairseq/models/xlmr.large.tar.gz -tar -xzvf xlmr.large.tar.gz - -# Load the model in fairseq -from fairseq.models.roberta import XLMRModel -xlmr = XLMRModel.from_pretrained('/path/to/xlmr.large', checkpoint_file='model.pt') -xlmr.eval() # disable dropout (or leave in train mode to finetune) -``` - -##### Apply sentence-piece-model (SPM) encoding to input text: -```python -en_tokens = xlmr.encode('Hello world!') -assert en_tokens.tolist() == [0, 35378, 8999, 38, 2] -xlmr.decode(en_tokens) # 'Hello world!' - -zh_tokens = xlmr.encode('你好，世界') -assert zh_tokens.tolist() == [0, 6, 124084, 4, 3221, 2] -xlmr.decode(zh_tokens) # '你好，世界' - -hi_tokens = xlmr.encode('नमस्ते दुनिया') -assert hi_tokens.tolist() == [0, 68700, 97883, 29405, 2] -xlmr.decode(hi_tokens) # 'नमस्ते दुनिया' - -ar_tokens = xlmr.encode('مرحبا بالعالم') -assert ar_tokens.tolist() == [0, 665, 193478, 258, 1705, 77796, 2] -xlmr.decode(ar_tokens) # 'مرحبا بالعالم' - -fr_tokens = xlmr.encode('Bonjour le monde') -assert fr_tokens.tolist() == [0, 84602, 95, 11146, 2] -xlmr.decode(fr_tokens) # 'Bonjour le monde' -``` - -##### Extract features from XLM-R: -```python -# Extract the last layer's features -last_layer_features = xlmr.extract_features(zh_tokens) -assert last_layer_features.size() == torch.Size([1, 6, 1024]) - -# Extract all layer's features (layer 0 is the embedding layer) -all_layers = xlmr.extract_features(zh_tokens, return_all_hiddens=True) -assert len(all_layers) == 25 -assert torch.all(all_layers[-1] == last_layer_features) -``` - -## Citation - -```bibtex -@article{conneau2019unsupervised, - title={Unsupervised Cross-lingual Representation Learning at Scale}, - author={Conneau, Alexis and Khandelwal, Kartikay and Goyal, Naman and Chaudhary, Vishrav and Wenzek, Guillaume and Guzm{\'a}n, Francisco and Grave, Edouard and Ott, Myle and Zettlemoyer, Luke and Stoyanov, Veselin}, - journal={arXiv preprint arXiv:1911.02116}, - year={2019} -} -``` - - -```bibtex -@article{goyal2021larger, - title={Larger-Scale Transformers for Multilingual Masked Language Modeling}, - author={Goyal, Naman and Du, Jingfei and Ott, Myle and Anantharaman, Giri and Conneau, Alexis}, - journal={arXiv preprint arXiv:2105.00572}, - year={2021} -} -``` diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/xmod/README.md b/audio/safeear/safeear_repo/fairseq_ours/examples/xmod/README.md deleted file mode 100644 index 46958b81410c1cd2f9ca1ddb044467d859700c63..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/xmod/README.md +++ /dev/null @@ -1,151 +0,0 @@ -# X-MOD: Lifting the Curse of Multilinguality by Pre-training Modular Transformers - -https://arxiv.org/abs/2205.06266 - - -## Introduction - -X-MOD extends multilingual masked language models like XLM-R to include language-specific modular components, introduced at each transformer layer. Each module is only used by one language. For fine-tuning, the modular components are frozen, and replaced with the target language in cross-lingual transfer settings. - - -## Pre-trained models - -Model | Size | # train steps | # langs | Download ----|---|---|---|--- -`xmod.base.13.125k` | BERT-base | 125k | 13 | [xmod.base.13.125k.tar.gz](https://dl.fbaipublicfiles.com/fairseq/models/xmod/xmod.base.13.125k.tar.gz) -`xmod.base.30.125k` | BERT-base | 125k | 30 | [xmod.base.30.125k.tar.gz](https://dl.fbaipublicfiles.com/fairseq/models/xmod/xmod.base.30.125k.tar.gz) -`xmod.base.30.195k` | BERT-base | 195k | 30 | [xmod.base.30.195k.tar.gz](https://dl.fbaipublicfiles.com/fairseq/models/xmod/xmod.base.30.195k.tar.gz) -`xmod.base.60.125k` | BERT-base | 125k | 60 | [xmod.base.60.125k.tar.gz](https://dl.fbaipublicfiles.com/fairseq/models/xmod/xmod.base.60.125k.tar.gz) -`xmod.base.60.265k` | BERT-base | 265k | 60 | [xmod.base.60.265k.tar.gz](https://dl.fbaipublicfiles.com/fairseq/models/xmod/xmod.base.60.265k.tar.gz) -`xmod.base.75.125k` | BERT-base | 125k | 75 | [xmod.base.75.125k.tar.gz](https://dl.fbaipublicfiles.com/fairseq/models/xmod/xmod.base.75.125k.tar.gz) -`xmod.base.75.269k` | BERT-base | 269k | 75 | [xmod.base.75.269k.tar.gz](https://dl.fbaipublicfiles.com/fairseq/models/xmod/xmod.base.75.269k.tar.gz) -`xmod.base` | BERT-base | 1M | 81 | [xmod.base.81.1M.tar.gz](https://dl.fbaipublicfiles.com/fairseq/models/xmod/xmod.base.81.1M.tar.gz) -`xmod.large.prenorm` | BERT-large | 500k | 81 | [xmod.large.prenorm.81.500k.tar.gz](https://dl.fbaipublicfiles.com/fairseq/models/xmod/xmod.large.prenorm.81.500k.tar.gz) - - -## Fine-tuning on NLI - -We next provide an example of how to fine-tune the pre-trained models above on Natural Language Inference (NLI). We use MNLI for training in English, and show how to run inference in other languages. - -### 1) Download a pre-trained model - -```bash -MODEL=xmod.base.81.1M -wget https://dl.fbaipublicfiles.com/fairseq/models/xmod/$MODEL.tar.gz -tar -xzf $MODEL.tar.gz -``` - -### 2) Download and preprocess [MNLI](https://cims.nyu.edu/~sbowman/multinli/) -```bash -wget https://cims.nyu.edu/~sbowman/multinli/multinli_1.0.zip -unzip multinli_1.0.zip -python ./examples/xmod/preprocess_nli.py \ - --sentencepiece-model $MODEL/sentencepiece.bpe.model \ - --train multinli_1.0/multinli_1.0_train.jsonl \ - --valid multinli_1.0/multinli_1.0_dev_matched.jsonl \ - --destdir multinli_1.0/fairseq -``` - -### 3) Fine-tune on MNLI: - -```bash -MAX_EPOCH=5 -LR=1e-05 -BATCH_SIZE=32 -DATA_DIR=multinli_1.0/fairseq/bin - -CUDA_VISIBLE_DEVICES=0 fairseq-train $DATA_DIR \ - --restore-file $MODEL/model.pt \ - --save-dir $MODEL/nli \ - --reset-optimizer \ - --reset-dataloader \ - --reset-meters \ - --best-checkpoint-metric accuracy \ - --maximize-best-checkpoint-metric \ - --task sentence_prediction_adapters \ - --num-classes 3 \ - --init-token 0 \ - --separator-token 2 \ - --max-positions 512 \ - --shorten-method "truncate" \ - --arch xmod_base \ - --dropout 0.1 \ - --attention-dropout 0.1 \ - --weight-decay 0.01 \ - --criterion sentence_prediction_adapters \ - --optimizer adam \ - --adam-betas '(0.9, 0.98)' \ - --adam-eps 1e-06 \ - --clip-norm 0.0 \ - --lr-scheduler fixed \ - --lr $LR \ - --fp16 \ - --fp16-init-scale 4 \ - --threshold-loss-scale 1 \ - --fp16-scale-window 128 \ - --batch-size $BATCH_SIZE \ - --required-batch-size-multiple 1 \ - --update-freq 1 \ - --max-epoch $MAX_EPOCH -``` - -### 4) Run inference - -After training the model, we can load it and run inference in our target language. The default language is set to English, which is why we were not required to pass a language ID to the model during fine-tuning. To run inference in a non-English language, we need to tell the model that the module of the target language should be used instead: - -```python -from fairseq.models.xmod import XMODModel - -MODEL='xmod.base.81.1M/nli' -DATA='multinli_1.0/fairseq/bin' - -# Load model -model = XMODModel.from_pretrained( - model_name_or_path=MODEL, - checkpoint_file='checkpoint_best.pt', - data_name_or_path=DATA, - suffix='', - criterion='cross_entropy', - bpe='sentencepiece', - sentencepiece_model=DATA+'/input0/sentencepiece.bpe.model') -model = model.eval(); # disable dropout -model = model.half(); # use FP16 -model = model.cuda(); # move to GPU - -def predict(premise, hypothesis, lang): - tokens = model.encode(premise, hypothesis) - idx = model.predict('sentence_classification_head', tokens, lang_id=[lang]).argmax().item() - dictionary = model.task.label_dictionary - return dictionary[idx + dictionary.nspecial] - -predict( - premise='X-Mod hat spezifische Module die für jede Sprache existieren.', - hypothesis='X-Mod hat Module.', - lang='de_DE' -) # entailment - -predict( - premise='Londres es la capital del Reino Unido.', - hypothesis='Londres está en Francia.', - lang='es_XX', -) # contradiction - -predict( - premise='Patxik gogoko ditu babarrunak.', - hypothesis='Patxik babarrunak bazkaldu zituen.', - lang='eu_ES', -) # neutral -``` - - -## Citation - -```bibtex -@misc{pfeiffer2022xmod, - doi = {10.48550/ARXIV.2205.06266}, - url = {https://arxiv.org/abs/2205.06266}, - title = {Lifting the Curse of Multilinguality by Pre-training Modular Transformers}, - publisher = {arXiv}, - year = {2022}, -} -``` diff --git a/audio/safeear/safeear_repo/fairseq_ours/examples/xmod/preprocess_nli.py b/audio/safeear/safeear_repo/fairseq_ours/examples/xmod/preprocess_nli.py deleted file mode 100644 index ed0b5d65752d3e709852f3384279d96274e4d25a..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/examples/xmod/preprocess_nli.py +++ /dev/null @@ -1,169 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import collections -import json -import os -import shutil -import subprocess -import sys -import tempfile -from multiprocessing import Pool - -import sentencepiece as spm - - -def preprocess(spm_model_path, train_path, valid_path, test_path, dest_dir, remove_empty=False, output_format='piece', workers=20): - with tempfile.TemporaryDirectory() as tmp: - # Tokenize with SentencePiece - for split, path in ('train', train_path), ('valid', valid_path), ('test', test_path): - if path is None: - continue - if path == '-': - path = sys.stdin.fileno() - with open(path, encoding='utf-8', errors='surrogateescape') as fin: - with open(f'{tmp}/{split}', mode='w', encoding='utf-8', errors='surrogateescape') as fout: - encoder = MultiprocessingEncoder(model=spm_model_path, remove_empty=remove_empty, output_format=output_format) - pool = Pool(workers, initializer=encoder.initializer) - encoded_lines = pool.imap(encoder.encode, fin, 10000) - for i, line in enumerate(encoded_lines, start=1): - if line is not None: - print(line, file=fout) - if i % 10000 == 0: - print("tokenized {} lines".format(i), file=sys.stderr) - - # Generate dictionary - sp = spm.SentencePieceProcessor(model_file=spm_model_path) - if output_format == 'piece': - vocab = [sp.id_to_piece(i) for i in range(3, sp.vocab_size())] - else: - vocab = map(str, range(sp.vocab_size())) - with open(f'{tmp}/dict.txt', mode='w', encoding='utf-8', errors='surrogateescape') as f: - for word in vocab: - print(word, 1, file=f) - - # Binarize - command = [ - 'python3', '-m', 'fairseq_cli.preprocess', - '--only-source', - '--thresholdsrc', '0', - '--destdir', dest_dir, - '--srcdict', f'{tmp}/dict.txt', - '--workers', '20', - ] - for split, path in ('train', train_path), ('valid', valid_path), ('test', test_path): - if path is not None: - command += [f'--{split}pref', f'{tmp}/{split}'] - subprocess.run(command) - - # Copy SentencePiece model - shutil.copyfile(spm_model_path, f'{dest_dir}/sentencepiece.bpe.model') - - -class MultiprocessingEncoder(object): - def __init__(self, model, remove_empty, output_format): - self.model = model - self.remove_empty = remove_empty - self.output_format = output_format - - def initializer(self): - global sp - sp = spm.SentencePieceProcessor(model_file=self.model) - - def encode(self, line): - global sp - line = line.strip() - if len(line) == 0 and self.remove_empty: - return None - - if self.output_format == 'piece': - return ' '.join(sp.encode_as_pieces(line)) - else: - return ' '.join(map(str, sp.encode(line))) - - -def write_lines(lines, path): - with open(path, mode='x', encoding='utf-8') as f: - for line in lines: - print(line, file=f) - - -def read_jsonl(path): - with open(path, encoding='utf-8') as f: - return [json.loads(line) for line in f.read().splitlines()] - - -def read_nli(path, langs=None): - data = read_jsonl(path) - - if langs is not None: - data = [sample for sample in data if sample.get('language') in langs] - - lang2count = collections.defaultdict(int) - for sample in data: - lang2count[sample.get('language')] += 1 - - if langs: - assert set(lang2count.keys()) == set(langs) - - nlangs = len(lang2count) - assert nlangs > 0 - lens = list(lang2count.values()) - assert all([lens[0] == length for length in lens]) - - print(f'Loaded {lens[0]} samples in {nlangs} languages from {path}', file=sys.stderr) - return data - - -def main(): - parser = argparse.ArgumentParser(description='Tokenize and binarize NLI data') - parser.add_argument('--sentencepiece-model', required=True) - parser.add_argument('--train', required=True, help='Training data in jsonl format') - parser.add_argument('--valid', required=True, help='Validation data in jsonl format') - parser.add_argument('--destdir', required=True) - - args = parser.parse_args() - - os.makedirs(args.destdir + '/raw',) - os.makedirs(args.destdir + '/bin', ) - - # Extract input/labels - for split, path in ('train', args.train), ('valid', args.valid): - data = read_nli(path, langs=None) - original_size = len(data) - data = [sample for sample in data if sample['gold_label'] != '-'] - assert all(sample['gold_label'] in ('contradiction', 'entailment', 'neutral') for sample in data) - filtered_size = len(data) - if filtered_size != original_size: - print(f'Filtered {filtered_size}/{original_size} samples from {path}', file=sys.stderr) - for name, field in ('input0', 'sentence1'), ('input1', 'sentence2'), ('label', 'gold_label'): - write_lines([sample[field] for sample in data], f'{args.destdir}/raw/{split}.{name}.txt') - - # Tokenize and binarize input - for field in 'input0', 'input1': - preprocess( - spm_model_path=args.sentencepiece_model, - train_path=f'{args.destdir}/raw/train.{field}.txt', - valid_path=f'{args.destdir}/raw/valid.{field}.txt', - test_path=None, - dest_dir=f'{args.destdir}/bin/{field}', - workers=20, - ) - - # Binarize labels - subprocess.run([ - 'python3', '-m', 'fairseq_cli.preprocess', - '--trainpref', f'{args.destdir}/raw/train.label.txt', - '--validpref', f'{args.destdir}/raw/valid.label.txt', - '--only-source', - '--thresholdsrc', '0', - '--destdir', f'{args.destdir}/bin/label', - '--workers', '20', - ]) - - -if __name__ == '__main__': - main() diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/__init__.py deleted file mode 100644 index 080c988b2da326c2fe356630d5641d367b37a546..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/__init__.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. -"""isort:skip_file""" - -import os -import sys - -try: - from .version import __version__ # noqa -except ImportError: - version_txt = os.path.join(os.path.dirname(__file__), "version.txt") - with open(version_txt) as f: - __version__ = f.read().strip() - -__all__ = ["pdb"] - -# backwards compatibility to support `from fairseq.X import Y` -from fairseq.distributed import utils as distributed_utils -from fairseq.logging import meters, metrics, progress_bar # noqa - -sys.modules["fairseq.distributed_utils"] = distributed_utils -sys.modules["fairseq.meters"] = meters -sys.modules["fairseq.metrics"] = metrics -sys.modules["fairseq.progress_bar"] = progress_bar - -# initialize hydra -from fairseq.dataclass.initialize import hydra_init - -hydra_init() - -import fairseq.criterions # noqa -import fairseq.distributed # noqa -import fairseq.models # noqa -import fairseq.modules # noqa -import fairseq.optim # noqa -import fairseq.optim.lr_scheduler # noqa -import fairseq.pdb # noqa -import fairseq.scoring # noqa -import fairseq.tasks # noqa -import fairseq.token_generation_constraints # noqa - -import fairseq.benchmark # noqa -import fairseq.model_parallel # noqa diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/benchmark/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/benchmark/__init__.py deleted file mode 100644 index 0317d5c623778fe40b7bf07b77769cd10c243244..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/benchmark/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -# import models/tasks to register them -from . import dummy_dataset, dummy_lm, dummy_masked_lm, dummy_model, dummy_mt # noqa diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/benchmark/benchmark_multihead_attention.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/benchmark/benchmark_multihead_attention.py deleted file mode 100644 index 8db8875fdf828fa3f2ba326bab94a227324f6011..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/benchmark/benchmark_multihead_attention.py +++ /dev/null @@ -1,171 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import itertools -import random - -import torch -from fairseq.modules.multihead_attention import MultiheadAttention -from torch.utils import benchmark - -BATCH = [20, 41, 97] -SEQ = 64 -EMB = 48 -HEADS = 4 -DROP = 0.1 -DEVICE = torch.device("cuda") -ATTN_MASK_DTYPE = [torch.uint8, torch.bool, torch.float] -KEY_PADDING_MASK_DTYPE = [torch.uint8, torch.bool] - - -def _reset_seeds(): - torch.manual_seed(0) - random.seed(0) - - -def _get_mask(to_dtype: torch.dtype, dim0: int, dim1: int): - if to_dtype == torch.float: - mask = torch.randint(0, 2, (dim0, dim1)).to(dtype=torch.bool) - return mask.to(dtype=to_dtype).masked_fill(mask, -float("inf")) - return torch.randint(0, 2, (dim0, dim1)).to(dtype=to_dtype) - - -def benchmark_multihead_attention( - label="", - attn_dtype=torch.uint8, - key_padding_dtype=torch.uint8, - add_bias_kv=False, - add_zero_attn=False, - static_kv=False, - batch_size=20, - embedding=EMB, - seq_len=SEQ, - num_heads=HEADS, -): - - results = [] - # device = torch.device("cuda") - - xformers_att_config = '{"name": "scaled_dot_product"}' - - attn_mask = _get_mask(to_dtype=attn_dtype, dim0=seq_len, dim1=seq_len) - key_padding_mask = _get_mask( - to_dtype=key_padding_dtype, dim0=batch_size, dim1=seq_len - ) - - q = torch.rand(seq_len, batch_size, embedding, requires_grad=True) - k = torch.rand(seq_len, batch_size, embedding, requires_grad=True) - v = torch.rand(seq_len, batch_size, embedding, requires_grad=True) - - _reset_seeds() - - original_mha = MultiheadAttention( - embedding, - num_heads, - dropout=0.0, - xformers_att_config=None, - add_bias_kv=add_bias_kv, - add_zero_attn=add_zero_attn, - ) - - xformers_mha = MultiheadAttention( - embedding, - num_heads, - dropout=0.0, - xformers_att_config=xformers_att_config, - add_bias_kv=add_bias_kv, - add_zero_attn=add_zero_attn, - ) - - def original_bench_fw(q, k, v, key_padding_mask, attn_mask, static_kv): - original_mha( - query=q, - key=k, - value=v, - key_padding_mask=key_padding_mask, - attn_mask=attn_mask, - static_kv=static_kv, - ) - - def xformers_bench_fw(q, k, v, key_padding_mask, attn_mask, static_kv): - xformers_mha( - query=q, - key=k, - value=v, - key_padding_mask=key_padding_mask, - attn_mask=attn_mask, - static_kv=static_kv, - ) - - def original_bench_fw_bw(q, k, v, key_padding_mask, attn_mask, static_kv): - output, _ = original_mha( - query=q, - key=k, - value=v, - key_padding_mask=key_padding_mask, - attn_mask=attn_mask, - static_kv=static_kv, - ) - loss = torch.norm(output) - loss.backward() - - def xformers_bench_fw_bw(q, k, v, key_padding_mask, attn_mask, static_kv): - output, _ = xformers_mha( - query=q, - key=k, - value=v, - key_padding_mask=key_padding_mask, - attn_mask=attn_mask, - static_kv=static_kv, - ) - loss = torch.norm(output) - loss.backward() - - fns = [ - original_bench_fw, - xformers_bench_fw, - original_bench_fw_bw, - xformers_bench_fw_bw, - ] - - for fn in fns: - results.append( - benchmark.Timer( - stmt="fn(q, k, v, key_padding_mask, attn_mask, static_kv)", - globals={ - "q": q, - "k": k, - "v": v, - "key_padding_mask": key_padding_mask, - "attn_mask": attn_mask, - "static_kv": static_kv, - "fn": fn, - }, - label="multihead fw + bw", - sub_label=f"{fn.__name__}", - description=label, - ).blocked_autorange(min_run_time=1) - ) - - compare = benchmark.Compare(results) - compare.print() - - -def run_benchmarks(): - for attn_dtype, key_padding_dtype, add_bias_kv, add_zero_attn in itertools.product( - ATTN_MASK_DTYPE, KEY_PADDING_MASK_DTYPE, [True, False], [True, False] - ): - label = f"attn_dtype {attn_dtype}, key_padding_dtype {key_padding_dtype}, \ - add_bias_kv {add_bias_kv}, add_zero_attn {add_zero_attn}" - benchmark_multihead_attention( - label=label, - attn_dtype=attn_dtype, - key_padding_dtype=key_padding_dtype, - add_bias_kv=add_bias_kv, - add_zero_attn=add_zero_attn, - ) - - -run_benchmarks() diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/benchmark/dummy_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/benchmark/dummy_dataset.py deleted file mode 100644 index 2f051754af55966e26850e94c121e0ff439bfd28..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/benchmark/dummy_dataset.py +++ /dev/null @@ -1,36 +0,0 @@ -import numpy as np -from fairseq.data import FairseqDataset - - -class DummyDataset(FairseqDataset): - def __init__(self, batch, num_items, item_size): - super().__init__() - self.batch = batch - self.num_items = num_items - self.item_size = item_size - - def __getitem__(self, index): - return index - - def __len__(self): - return self.num_items - - def collater(self, samples): - return self.batch - - @property - def sizes(self): - return np.array([self.item_size] * self.num_items) - - def num_tokens(self, index): - return self.item_size - - def size(self, index): - return self.item_size - - def ordered_indices(self): - return np.arange(self.num_items) - - @property - def supports_prefetch(self): - return False diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/benchmark/dummy_lm.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/benchmark/dummy_lm.py deleted file mode 100644 index 2a24a73d1cbcb97701a9402c9404cf3357bd2f2a..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/benchmark/dummy_lm.py +++ /dev/null @@ -1,83 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import logging -from dataclasses import dataclass, field -from typing import Optional - -import torch -from fairseq.data import Dictionary -from fairseq.dataclass import FairseqDataclass -from fairseq.tasks import FairseqTask, register_task -from omegaconf import II - -from .dummy_dataset import DummyDataset - -logger = logging.getLogger(__name__) - - -@dataclass -class DummyLMConfig(FairseqDataclass): - dict_size: int = 49996 - dataset_size: int = 100000 - tokens_per_sample: int = field( - default=512, metadata={"help": "max sequence length"} - ) - add_bos_token: bool = False - batch_size: Optional[int] = II("dataset.batch_size") - max_tokens: Optional[int] = II("dataset.max_tokens") - max_target_positions: int = II("task.tokens_per_sample") - - -@register_task("dummy_lm", dataclass=DummyLMConfig) -class DummyLMTask(FairseqTask): - def __init__(self, cfg: DummyLMConfig): - super().__init__(cfg) - - # load dictionary - self.dictionary = Dictionary() - for i in range(cfg.dict_size): - self.dictionary.add_symbol("word{}".format(i)) - self.dictionary.pad_to_multiple_(8) # often faster if divisible by 8 - logger.info("dictionary: {} types".format(len(self.dictionary))) - - seq = torch.arange(cfg.tokens_per_sample + 1) + self.dictionary.pad() + 1 - - self.dummy_src = seq[:-1] - self.dummy_tgt = seq[1:] - - def load_dataset(self, split, epoch=1, combine=False, **kwargs): - """Load a given dataset split. - Args: - split (str): name of the split (e.g., train, valid, test) - """ - if self.cfg.batch_size is not None: - bsz = self.cfg.batch_size - else: - bsz = max(1, self.cfg.max_tokens // self.cfg.tokens_per_sample) - self.datasets[split] = DummyDataset( - { - "id": 1, - "net_input": { - "src_tokens": torch.stack([self.dummy_src for _ in range(bsz)]), - "src_lengths": torch.full( - (bsz,), self.cfg.tokens_per_sample, dtype=torch.long - ), - }, - "target": torch.stack([self.dummy_tgt for _ in range(bsz)]), - "nsentences": bsz, - "ntokens": bsz * self.cfg.tokens_per_sample, - }, - num_items=self.cfg.dataset_size, - item_size=self.cfg.tokens_per_sample, - ) - - @property - def source_dictionary(self): - return self.dictionary - - @property - def target_dictionary(self): - return self.dictionary diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/benchmark/dummy_masked_lm.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/benchmark/dummy_masked_lm.py deleted file mode 100644 index 84c01dbe4749d130e0ec71ded2ac6f5371bc71c5..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/benchmark/dummy_masked_lm.py +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import logging -from dataclasses import dataclass, field -from typing import Optional - -import torch -from fairseq.data import Dictionary -from fairseq.dataclass import FairseqDataclass -from fairseq.tasks import FairseqTask, register_task -from omegaconf import II - -from .dummy_dataset import DummyDataset - -logger = logging.getLogger(__name__) - - -@dataclass -class DummyMaskedLMConfig(FairseqDataclass): - dict_size: int = 49996 - dataset_size: int = 100000 - tokens_per_sample: int = field( - default=512, - metadata={ - "help": "max number of total tokens over all" - " segments per sample for BERT dataset" - }, - ) - batch_size: Optional[int] = II("dataset.batch_size") - max_tokens: Optional[int] = II("dataset.max_tokens") - max_target_positions: int = II("task.tokens_per_sample") - - -@register_task("dummy_masked_lm", dataclass=DummyMaskedLMConfig) -class DummyMaskedLMTask(FairseqTask): - def __init__(self, cfg: DummyMaskedLMConfig): - super().__init__(cfg) - - self.dictionary = Dictionary() - for i in range(cfg.dict_size): - self.dictionary.add_symbol("word{}".format(i)) - logger.info("dictionary: {} types".format(len(self.dictionary))) - # add mask token - self.mask_idx = self.dictionary.add_symbol("<mask>") - self.dictionary.pad_to_multiple_(8) # often faster if divisible by 8 - - mask_idx = 0 - pad_idx = 1 - seq = torch.arange(cfg.tokens_per_sample) + pad_idx + 1 - mask = torch.arange(2, cfg.tokens_per_sample, 7) # ~15% - src = seq.clone() - src[mask] = mask_idx - tgt = torch.full_like(seq, pad_idx) - tgt[mask] = seq[mask] - - self.dummy_src = src - self.dummy_tgt = tgt - - def load_dataset(self, split, epoch=1, combine=False, **kwargs): - """Load a given dataset split. - Args: - split (str): name of the split (e.g., train, valid, test) - """ - if self.cfg.batch_size is not None: - bsz = self.cfg.batch_size - else: - bsz = max(1, self.cfg.max_tokens // self.cfg.tokens_per_sample) - self.datasets[split] = DummyDataset( - { - "id": 1, - "net_input": { - "src_tokens": torch.stack([self.dummy_src for _ in range(bsz)]), - "src_lengths": torch.full( - (bsz,), self.cfg.tokens_per_sample, dtype=torch.long - ), - }, - "target": torch.stack([self.dummy_tgt for _ in range(bsz)]), - "nsentences": bsz, - "ntokens": bsz * self.cfg.tokens_per_sample, - }, - num_items=self.cfg.dataset_size, - item_size=self.cfg.tokens_per_sample, - ) - - @property - def source_dictionary(self): - return self.dictionary - - @property - def target_dictionary(self): - return self.dictionary diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/benchmark/dummy_model.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/benchmark/dummy_model.py deleted file mode 100644 index ff26e4fe655d8e8d7f9942c4bd3df7cd267405fb..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/benchmark/dummy_model.py +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import torch.nn as nn -import torch.nn.functional as F -from fairseq.data import Dictionary -from fairseq.models import ( - FairseqDecoder, - FairseqLanguageModel, - register_model, - register_model_architecture, -) - - -@register_model("dummy_model") -class DummyModel(FairseqLanguageModel): - def __init__(self, args, encoder): - super().__init__(encoder) - self.args = args - - @staticmethod - def add_args(parser): - parser.add_argument("--num-layers", type=int, default=24) - parser.add_argument("--embed-dim", type=int, default=1024) - - @classmethod - def build_model(cls, args, task): - encoder = DummyEncoder( - num_embed=len(task.target_dictionary), - embed_dim=args.embed_dim, - num_layers=args.num_layers, - ) - return cls(args, encoder) - - def forward(self, src_tokens, masked_tokens=None, **kwargs): - return self.decoder(src_tokens, masked_tokens=masked_tokens) - - -class DummyEncoder(FairseqDecoder): - def __init__(self, num_embed=50000, embed_dim=1024, num_layers=24): - super().__init__(Dictionary()) - self.embed = nn.Embedding( - num_embeddings=num_embed, embedding_dim=embed_dim, padding_idx=0 - ) - self.layers_a = nn.ModuleList( - [ - nn.Sequential( - nn.LayerNorm(embed_dim), - nn.Linear(embed_dim, 3 * embed_dim), # q, k, v input projection - nn.Linear(3 * embed_dim, embed_dim), # skip self-attention - nn.Linear(embed_dim, embed_dim), # output projection - nn.Dropout(), - ) - for i in range(num_layers) - ] - ) - self.layers_b = nn.ModuleList( - [ - nn.Sequential( - nn.LayerNorm(embed_dim), - nn.Linear(embed_dim, 4 * embed_dim), # FFN - nn.ReLU(), - nn.Linear(4 * embed_dim, embed_dim), # FFN - nn.Dropout(0.1), - ) - for i in range(num_layers) - ] - ) - self.out_proj = nn.Linear(embed_dim, num_embed) - - def forward(self, tokens, masked_tokens=None): - x = self.embed(tokens) - for layer_a, layer_b in zip(self.layers_a, self.layers_b): - x = x + layer_a(x) - x = x + layer_b(x) - x = self.out_proj(x) - if masked_tokens is not None: - x = x[masked_tokens] - return (x,) - - def max_positions(self): - return 1024 - - def get_normalized_probs(self, net_output, log_probs, sample=None): - logits = net_output[0].float() - if log_probs: - return F.log_softmax(logits, dim=-1) - else: - return F.softmax(logits, dim=-1) - - -@register_model_architecture("dummy_model", "dummy_model") -def base_architecture(args): - pass diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/benchmark/dummy_mt.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/benchmark/dummy_mt.py deleted file mode 100644 index c056edcc2e397450f0286eefedec9d5dd4d776a8..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/benchmark/dummy_mt.py +++ /dev/null @@ -1,118 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import logging - -import numpy as np -import torch -from fairseq.data import Dictionary, FairseqDataset -from fairseq.tasks import LegacyFairseqTask, register_task - -logger = logging.getLogger(__name__) - - -@register_task("dummy_mt") -class DummyMTTask(LegacyFairseqTask): - @staticmethod - def add_args(parser): - """Add task-specific arguments to the parser.""" - parser.add_argument("--dict-size", default=49996, type=int) - parser.add_argument("--dataset-size", default=100000, type=int) - parser.add_argument("--src-len", default=30, type=int) - parser.add_argument("--tgt-len", default=30, type=int) - - def __init__(self, args, dictionary): - super().__init__(args) - self.dictionary = dictionary - self.seed = args.seed - - dictionary.pad_to_multiple_(8) # often faster if divisible by 8 - - self.dummy_src = torch.arange(args.src_len + 1) + dictionary.pad() + 1 - self.dummy_tgt = torch.arange(args.tgt_len + 1) + dictionary.pad() + 1 - - @classmethod - def setup_task(cls, args, **kwargs): - """Setup the task.""" - dictionary = Dictionary() - for i in range(args.dict_size): - dictionary.add_symbol("word{}".format(i)) - logger.info("dictionary: {} types".format(len(dictionary))) - - args.max_source_positions = args.src_len + dictionary.pad() + 2 - args.max_target_positions = args.tgt_len + dictionary.pad() + 2 - - return cls(args, dictionary) - - def load_dataset(self, split, epoch=1, combine=False, **kwargs): - """Load a given dataset split. - Args: - split (str): name of the split (e.g., train, valid, test) - """ - item_size = max(self.args.src_len, self.args.tgt_len) - if self.args.batch_size is not None: - bsz = self.args.batch_size - else: - bsz = max(1, self.args.max_tokens // item_size) - tgt = torch.stack([self.dummy_tgt for _ in range(bsz)]) - self.datasets[split] = DummyDataset( - { - "id": 1, - "net_input": { - "src_tokens": torch.stack([self.dummy_src for _ in range(bsz)]), - "src_lengths": torch.full( - (bsz,), self.args.src_len, dtype=torch.long - ), - "prev_output_tokens": tgt.clone(), - }, - "target": tgt, - "nsentences": bsz, - "ntokens": bsz * self.args.tgt_len, - }, - num_items=self.args.dataset_size, - item_size=item_size, - ) - - @property - def source_dictionary(self): - return self.dictionary - - @property - def target_dictionary(self): - return self.dictionary - - -class DummyDataset(FairseqDataset): - def __init__(self, batch, num_items, item_size): - super().__init__() - self.batch = batch - self.num_items = num_items - self.item_size = item_size - - def __getitem__(self, index): - return index - - def __len__(self): - return self.num_items - - def collater(self, samples): - return self.batch - - @property - def sizes(self): - return np.array([self.item_size] * self.num_items) - - def num_tokens(self, index): - return self.item_size - - def size(self, index): - return self.item_size - - def ordered_indices(self): - return np.arange(self.num_items) - - @property - def supports_prefetch(self): - return False diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/binarizer.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/binarizer.py deleted file mode 100644 index b2585edab96021466f04fc6b156bc9b1bec50dd0..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/binarizer.py +++ /dev/null @@ -1,380 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import logging -import os -import typing as tp -from abc import ABC, abstractmethod -from collections import Counter -from dataclasses import dataclass -from multiprocessing import Pool - -import torch -from fairseq.data import Dictionary, indexed_dataset -from fairseq.file_chunker_utils import Chunker, find_offsets -from fairseq.file_io import PathManager -from fairseq.tokenizer import tokenize_line - -logger = logging.getLogger("binarizer") - - -@dataclass -class BinarizeSummary: - """ - Keep track of what's going on in the binarizer - """ - - num_seq: int = 0 - replaced: tp.Optional[Counter] = None - num_tok: int = 0 - - @property - def num_replaced(self) -> int: - if self.replaced is None: - return 0 - return sum(self.replaced.values()) - - @property - def replaced_percent(self) -> float: - return 100 * self.num_replaced / self.num_tok - - def __str__(self) -> str: - base = f"{self.num_seq} sents, {self.num_tok} tokens" - if self.replaced is None: - return base - - return f"{base}, {self.replaced_percent:.3}% replaced" - - def merge(self, other: "BinarizeSummary"): - replaced = None - if self.replaced is not None: - replaced = self.replaced - if other.replaced is not None: - if replaced is None: - replaced = other.replaced - else: - replaced += other.replaced - self.replaced = replaced - self.num_seq += other.num_seq - self.num_tok += other.num_tok - - -class Binarizer(ABC): - """ - a binarizer describes how to take a string and build a tensor out of it - """ - - @abstractmethod - def binarize_line( - self, - line: str, - summary: BinarizeSummary, - ) -> torch.IntTensor: - ... - - -def _worker_prefix(output_prefix: str, worker_id: int): - return f"{output_prefix}.pt{worker_id}" - - -class FileBinarizer: - """ - An file binarizer can take a file, tokenize it, and binarize each line to a tensor - """ - - @classmethod - def multiprocess_dataset( - cls, - input_file: str, - dataset_impl: str, - binarizer: Binarizer, - output_prefix: str, - vocab_size=None, - num_workers=1, - ) -> BinarizeSummary: - final_summary = BinarizeSummary() - - offsets = find_offsets(input_file, num_workers) - # find_offsets returns a list of position [pos1, pos2, pos3, pos4] but we would want pairs: - # [(pos1, pos2), (pos2, pos3), (pos3, pos4)] to process the chunks with start/end info - # we zip the list with itself shifted by one to get all the pairs. - (first_chunk, *more_chunks) = zip(offsets, offsets[1:]) - pool = None - if num_workers > 1: - pool = Pool(processes=num_workers - 1) - worker_results = [ - pool.apply_async( - cls._binarize_chunk_and_finalize, - args=( - binarizer, - input_file, - start_offset, - end_offset, - _worker_prefix( - output_prefix, - worker_id, - ), - dataset_impl, - ), - kwds={ - "vocab_size": vocab_size, - } - if vocab_size is not None - else {}, - ) - for worker_id, (start_offset, end_offset) in enumerate( - more_chunks, start=1 - ) - ] - - pool.close() - pool.join() - for r in worker_results: - summ = r.get() - final_summary.merge(summ) - - # do not close the bin file as we need to merge the worker results in - final_ds, summ = cls._binarize_file_chunk( - binarizer, - input_file, - offset_start=first_chunk[0], - offset_end=first_chunk[1], - output_prefix=output_prefix, - dataset_impl=dataset_impl, - vocab_size=vocab_size if vocab_size is not None else None, - ) - final_summary.merge(summ) - - if num_workers > 1: - for worker_id in range(1, num_workers): - # merge the worker outputs - worker_output_prefix = _worker_prefix( - output_prefix, - worker_id, - ) - final_ds.merge_file_(worker_output_prefix) - try: - os.remove(indexed_dataset.data_file_path(worker_output_prefix)) - os.remove(indexed_dataset.index_file_path(worker_output_prefix)) - except Exception as e: - logger.error( - f"couldn't remove {worker_output_prefix}.*", exc_info=e - ) - - # now we can close the file - idx_file = indexed_dataset.index_file_path(output_prefix) - final_ds.finalize(idx_file) - return final_summary - - @staticmethod - def _binarize_file_chunk( - binarizer: Binarizer, - filename: str, - offset_start: int, - offset_end: int, - output_prefix: str, - dataset_impl: str, - vocab_size=None, - ) -> tp.Tuple[tp.Any, BinarizeSummary]: # (dataset builder, BinarizeSummary) - """ - creates a dataset builder and append binarized items to it. This function does not - finalize the builder, this is useful if you want to do other things with your bin file - like appending/merging other files - """ - bin_file = indexed_dataset.data_file_path(output_prefix) - ds = indexed_dataset.make_builder( - bin_file, - impl=dataset_impl, - vocab_size=vocab_size, - ) - summary = BinarizeSummary() - - with Chunker( - PathManager.get_local_path(filename), offset_start, offset_end - ) as line_iterator: - for line in line_iterator: - ds.add_item(binarizer.binarize_line(line, summary)) - - return ds, summary - - @classmethod - def _binarize_chunk_and_finalize( - cls, - binarizer: Binarizer, - filename: str, - offset_start: int, - offset_end: int, - output_prefix: str, - dataset_impl: str, - vocab_size=None, - ): - """ - same as above, but also finalizes the builder - """ - ds, summ = cls._binarize_file_chunk( - binarizer, - filename, - offset_start, - offset_end, - output_prefix, - dataset_impl, - vocab_size=vocab_size, - ) - - idx_file = indexed_dataset.index_file_path(output_prefix) - ds.finalize(idx_file) - - return summ - - -class VocabularyDatasetBinarizer(Binarizer): - """ - Takes a Dictionary/Vocabulary, assign ids to each - token using the dictionary encode_line function. - """ - - def __init__( - self, - dict: Dictionary, - tokenize: tp.Callable[[str], tp.List[str]] = tokenize_line, - append_eos: bool = True, - reverse_order: bool = False, - already_numberized: bool = False, - ) -> None: - self.dict = dict - self.tokenize = tokenize - self.append_eos = append_eos - self.reverse_order = reverse_order - self.already_numberized = already_numberized - super().__init__() - - def binarize_line( - self, - line: str, - summary: BinarizeSummary, - ): - if summary.replaced is None: - summary.replaced = Counter() - - def replaced_consumer(word, idx): - if idx == self.dict.unk_index and word != self.dict.unk_word: - summary.replaced.update([word]) - - if self.already_numberized: - id_strings = line.strip().split() - id_list = [int(id_string) for id_string in id_strings] - if self.reverse_order: - id_list.reverse() - if self.append_eos: - id_list.append(self.dict.eos()) - ids = torch.IntTensor(id_list) - else: - ids = self.dict.encode_line( - line=line, - line_tokenizer=self.tokenize, - add_if_not_exist=False, - consumer=replaced_consumer, - append_eos=self.append_eos, - reverse_order=self.reverse_order, - ) - - summary.num_seq += 1 - summary.num_tok += len(ids) - return ids - - -class AlignmentDatasetBinarizer(Binarizer): - """ - binarize by parsing a set of alignments and packing - them in a tensor (see utils.parse_alignment) - """ - - def __init__( - self, - alignment_parser: tp.Callable[[str], torch.IntTensor], - ) -> None: - super().__init__() - self.alignment_parser = alignment_parser - - def binarize_line( - self, - line: str, - summary: BinarizeSummary, - ): - ids = self.alignment_parser(line) - summary.num_seq += 1 - summary.num_tok += len(ids) - return ids - - -class LegacyBinarizer: - @classmethod - def binarize( - cls, - filename: str, - dico: Dictionary, - consumer: tp.Callable[[torch.IntTensor], None], - tokenize: tp.Callable[[str], tp.List[str]] = tokenize_line, - append_eos: bool = True, - reverse_order: bool = False, - offset: int = 0, - end: int = -1, - already_numberized: bool = False, - ) -> tp.Dict[str, int]: - binarizer = VocabularyDatasetBinarizer( - dict=dico, - tokenize=tokenize, - append_eos=append_eos, - reverse_order=reverse_order, - already_numberized=already_numberized, - ) - return cls._consume_file( - filename, - binarizer, - consumer, - offset_start=offset, - offset_end=end, - ) - - @classmethod - def binarize_alignments( - cls, - filename: str, - alignment_parser: tp.Callable[[str], torch.IntTensor], - consumer: tp.Callable[[torch.IntTensor], None], - offset: int = 0, - end: int = -1, - ) -> tp.Dict[str, int]: - binarizer = AlignmentDatasetBinarizer(alignment_parser) - return cls._consume_file( - filename, - binarizer, - consumer, - offset_start=offset, - offset_end=end, - ) - - @staticmethod - def _consume_file( - filename: str, - binarizer: Binarizer, - consumer: tp.Callable[[torch.IntTensor], None], - offset_start: int, - offset_end: int, - ) -> tp.Dict[str, int]: - summary = BinarizeSummary() - - with Chunker( - PathManager.get_local_path(filename), offset_start, offset_end - ) as line_iterator: - for line in line_iterator: - consumer(binarizer.binarize_line(line, summary)) - - return { - "nseq": summary.num_seq, - "nunk": summary.num_replaced, - "ntok": summary.num_tok, - "replaced": summary.replaced, - } diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/checkpoint_utils.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/checkpoint_utils.py deleted file mode 100644 index aeb04f79bee231e595086e6d39ec0a1839ab4402..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/checkpoint_utils.py +++ /dev/null @@ -1,901 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import ast -import collections -import contextlib -import inspect -import logging -import os -import re -import time -import traceback -from collections import OrderedDict -from pathlib import Path -from typing import Any, Dict, Optional, Union - -import numpy as np -import torch -from fairseq.data import data_utils -from fairseq.dataclass.configs import CheckpointConfig -from fairseq.dataclass.utils import ( - convert_namespace_to_omegaconf, - overwrite_args_by_name, -) -from fairseq.distributed.fully_sharded_data_parallel import FSDP, has_FSDP -from fairseq.file_io import PathManager -from fairseq.models import FairseqDecoder, FairseqEncoder -from omegaconf import DictConfig, OmegaConf, open_dict - -logger = logging.getLogger(__name__) - - -def save_checkpoint(cfg: CheckpointConfig, trainer, epoch_itr, val_loss): - from fairseq import meters - - # only one worker should attempt to create the required dir - if trainer.data_parallel_rank == 0: - os.makedirs(cfg.save_dir, exist_ok=True) - - prev_best = getattr(save_checkpoint, "best", val_loss) - if val_loss is not None: - best_function = max if cfg.maximize_best_checkpoint_metric else min - save_checkpoint.best = best_function(val_loss, prev_best) - - if cfg.no_save: - return - - trainer.consolidate_optimizer() # TODO(SS): do we need this if no_save_optimizer_state - - if not trainer.should_save_checkpoint_on_current_rank: - if trainer.always_call_state_dict_during_save_checkpoint: - trainer.state_dict() - return - - write_timer = meters.StopwatchMeter() - write_timer.start() - - epoch = epoch_itr.epoch - end_of_epoch = epoch_itr.end_of_epoch() - updates = trainer.get_num_updates() - - logger.info(f"Preparing to save checkpoint for epoch {epoch} @ {updates} updates") - - def is_better(a, b): - return a >= b if cfg.maximize_best_checkpoint_metric else a <= b - - suffix = trainer.checkpoint_suffix - checkpoint_conds = collections.OrderedDict() - checkpoint_conds["checkpoint{}{}.pt".format(epoch, suffix)] = ( - end_of_epoch and not cfg.no_epoch_checkpoints and epoch % cfg.save_interval == 0 - ) - checkpoint_conds["checkpoint_{}_{}{}.pt".format(epoch, updates, suffix)] = ( - not end_of_epoch - and cfg.save_interval_updates > 0 - and updates % cfg.save_interval_updates == 0 - ) - checkpoint_conds["checkpoint_best{}.pt".format(suffix)] = val_loss is not None and ( - not hasattr(save_checkpoint, "best") - or is_better(val_loss, save_checkpoint.best) - ) - if val_loss is not None and cfg.keep_best_checkpoints > 0: - worst_best = getattr(save_checkpoint, "best", None) - chkpts = checkpoint_paths( - cfg.save_dir, - pattern=r"checkpoint\.best_{}_(\d+\.?\d*){}\.pt".format( - cfg.best_checkpoint_metric, suffix - ), - ) - if len(chkpts) > 0: - p = chkpts[-1] if cfg.maximize_best_checkpoint_metric else chkpts[0] - worst_best = float(p.rsplit("_")[-1].replace("{}.pt".format(suffix), "")) - # add random digits to resolve ties - with data_utils.numpy_seed(epoch, updates, val_loss): - rand_sfx = np.random.randint(0, cfg.keep_best_checkpoints) - - checkpoint_conds[ - "checkpoint.best_{}_{:.3f}{}{}.pt".format( - cfg.best_checkpoint_metric, val_loss, rand_sfx, suffix - ) - ] = worst_best is None or is_better(val_loss, worst_best) - checkpoint_conds[ - "checkpoint_last{}.pt".format(suffix) - ] = not cfg.no_last_checkpoints - - extra_state = {"train_iterator": epoch_itr.state_dict(), "val_loss": val_loss} - if hasattr(save_checkpoint, "best"): - extra_state.update({"best": save_checkpoint.best}) - - checkpoints = [ - os.path.join(cfg.save_dir, fn) for fn, cond in checkpoint_conds.items() if cond - ] - if len(checkpoints) > 0 and trainer.should_save_checkpoint_on_current_rank: - trainer.save_checkpoint(checkpoints[0], extra_state) - for cp in checkpoints[1:]: - if cfg.write_checkpoints_asynchronously: - # TODO[ioPath]: Need to implement a delayed asynchronous - # file copying/moving feature. - logger.warning( - f"ioPath is not copying {checkpoints[0]} to {cp} " - "since async write mode is on." - ) - else: - assert PathManager.copy( - checkpoints[0], cp, overwrite=True - ), f"Failed to copy {checkpoints[0]} to {cp}" - - write_timer.stop() - logger.info( - "Saved checkpoint {} (epoch {} @ {} updates, score {}) (writing took {} seconds)".format( - checkpoints[0], epoch, updates, val_loss, write_timer.sum - ) - ) - - if not end_of_epoch and cfg.keep_interval_updates > 0: - # remove old checkpoints; checkpoints are sorted in descending order - if cfg.keep_interval_updates_pattern == -1: - checkpoints = checkpoint_paths( - cfg.save_dir, pattern=r"checkpoint_\d+_(\d+){}\.pt".format(suffix) - ) - else: - checkpoints = checkpoint_paths( - cfg.save_dir, - pattern=r"checkpoint_\d+_(\d+){}\.pt".format(suffix), - keep_match=True, - ) - checkpoints = [ - x[0] - for x in checkpoints - if x[1] % cfg.keep_interval_updates_pattern != 0 - ] - - for old_chk in checkpoints[cfg.keep_interval_updates :]: - if os.path.lexists(old_chk): - os.remove(old_chk) - elif PathManager.exists(old_chk): - PathManager.rm(old_chk) - - if cfg.keep_last_epochs > 0: - # remove old epoch checkpoints; checkpoints are sorted in descending order - checkpoints = checkpoint_paths( - cfg.save_dir, pattern=r"checkpoint(\d+){}\.pt".format(suffix) - ) - for old_chk in checkpoints[cfg.keep_last_epochs :]: - if os.path.lexists(old_chk): - os.remove(old_chk) - elif PathManager.exists(old_chk): - PathManager.rm(old_chk) - - if cfg.keep_best_checkpoints > 0: - # only keep the best N checkpoints according to validation metric - checkpoints = checkpoint_paths( - cfg.save_dir, - pattern=r"checkpoint\.best_{}_(\d+\.?\d*){}\.pt".format( - cfg.best_checkpoint_metric, suffix - ), - ) - if not cfg.maximize_best_checkpoint_metric: - checkpoints = checkpoints[::-1] - for old_chk in checkpoints[cfg.keep_best_checkpoints :]: - if os.path.lexists(old_chk): - os.remove(old_chk) - elif PathManager.exists(old_chk): - PathManager.rm(old_chk) - - -def load_checkpoint(cfg: CheckpointConfig, trainer, **passthrough_args): - """ - Load a checkpoint and restore the training iterator. - - *passthrough_args* will be passed through to - ``trainer.get_train_iterator``. - """ - - reset_optimizer = cfg.reset_optimizer - reset_lr_scheduler = cfg.reset_lr_scheduler - optimizer_overrides = ast.literal_eval(cfg.optimizer_overrides) - reset_meters = cfg.reset_meters - reset_dataloader = cfg.reset_dataloader - - if cfg.finetune_from_model is not None and ( - reset_optimizer or reset_lr_scheduler or reset_meters or reset_dataloader - ): - raise ValueError( - "--finetune-from-model can not be set together with either --reset-optimizer" - " or reset_lr_scheduler or reset_meters or reset_dataloader" - ) - - suffix = trainer.checkpoint_suffix - if ( - cfg.restore_file == "checkpoint_last.pt" - ): # default value of restore_file is 'checkpoint_last.pt' - checkpoint_path = os.path.join( - cfg.save_dir, "checkpoint_last{}.pt".format(suffix) - ) - first_launch = not PathManager.exists(checkpoint_path) - if first_launch and getattr(cfg, "continue_once", None) is not None: - checkpoint_path = cfg.continue_once - elif cfg.finetune_from_model is not None and first_launch: - # if there is no last checkpoint to restore, start the finetune from pretrained model - # else just use usual logic to load checkpoint, e.g. restart from last checkpoint and etc. - if PathManager.exists(cfg.finetune_from_model): - checkpoint_path = cfg.finetune_from_model - reset_optimizer = True - reset_lr_scheduler = True - reset_meters = True - reset_dataloader = True - logger.info( - f"loading pretrained model from {checkpoint_path}: " - "optimizer, lr scheduler, meters, dataloader will be reset" - ) - else: - raise ValueError( - f"--finetune-from-model {cfg.finetune_from_model} does not exist" - ) - elif suffix is not None: - checkpoint_path = cfg.restore_file.replace(".pt", suffix + ".pt") - else: - checkpoint_path = cfg.restore_file - - if cfg.restore_file != "checkpoint_last.pt" and cfg.finetune_from_model: - raise ValueError( - "--finetune-from-model and --restore-file (non-default value) " - "can not be specified together: " + str(cfg) - ) - - extra_state = trainer.load_checkpoint( - checkpoint_path, - reset_optimizer, - reset_lr_scheduler, - optimizer_overrides, - reset_meters=reset_meters, - ) - - if ( - extra_state is not None - and "best" in extra_state - and not reset_optimizer - and not reset_meters - ): - save_checkpoint.best = extra_state["best"] - - if extra_state is not None and not reset_dataloader: - # restore iterator from checkpoint - itr_state = extra_state["train_iterator"] - epoch_itr = trainer.get_train_iterator( - epoch=itr_state["epoch"], load_dataset=True, **passthrough_args - ) - epoch_itr.load_state_dict(itr_state) - else: - epoch_itr = trainer.get_train_iterator( - epoch=1, load_dataset=True, **passthrough_args - ) - - trainer.lr_step(epoch_itr.epoch) - - return extra_state, epoch_itr - - -def load_checkpoint_to_cpu(path, arg_overrides=None, load_on_all_ranks=False): - """Loads a checkpoint to CPU (with upgrading for backward compatibility). - - If doing single-GPU training or if the checkpoint is only being loaded by at - most one process on each node (current default behavior is for only rank 0 - to read the checkpoint from disk), load_on_all_ranks should be False to - avoid errors from torch.distributed not having been initialized or - torch.distributed.barrier() hanging. - - If all processes on each node may be loading the checkpoint - simultaneously, load_on_all_ranks should be set to True to avoid I/O - conflicts. - - There's currently no support for > 1 but < all processes loading the - checkpoint on each node. - """ - local_path = PathManager.get_local_path(path) - # The locally cached file returned by get_local_path() may be stale for - # remote files that are periodically updated/overwritten (ex: - # checkpoint_last.pt) - so we remove the local copy, sync across processes - # (if needed), and then download a fresh copy. - if local_path != path and PathManager.path_requires_pathmanager(path): - try: - os.remove(local_path) - except FileNotFoundError: - # With potentially multiple processes removing the same file, the - # file being missing is benign (missing_ok isn't available until - # Python 3.8). - pass - if load_on_all_ranks: - torch.distributed.barrier() - local_path = PathManager.get_local_path(path) - - with open(local_path, "rb") as f: - state = torch.load(f, map_location=torch.device("cpu")) - - if "args" in state and state["args"] is not None and arg_overrides is not None: - args = state["args"] - for arg_name, arg_val in arg_overrides.items(): - setattr(args, arg_name, arg_val) - - if "cfg" in state and state["cfg"] is not None: - - # hack to be able to set Namespace in dict config. this should be removed when we update to newer - # omegaconf version that supports object flags, or when we migrate all existing models - from omegaconf import _utils - - old_primitive = _utils.is_primitive_type - _utils.is_primitive_type = lambda _: True - - state["cfg"] = OmegaConf.create(state["cfg"]) - - _utils.is_primitive_type = old_primitive - OmegaConf.set_struct(state["cfg"], True) - - if arg_overrides is not None: - overwrite_args_by_name(state["cfg"], arg_overrides) - - state = _upgrade_state_dict(state) - return state - - -def load_model_ensemble( - filenames, - arg_overrides: Optional[Dict[str, Any]] = None, - task=None, - strict=True, - suffix="", - num_shards=1, - state=None, -): - """Loads an ensemble of models. - - Args: - filenames (List[str]): checkpoint files to load - arg_overrides (Dict[str,Any], optional): override model args that - were used during model training - task (fairseq.tasks.FairseqTask, optional): task to use for loading - """ - assert not ( - strict and num_shards > 1 - ), "Cannot load state dict with strict=True and checkpoint shards > 1" - ensemble, args, _task = load_model_ensemble_and_task( - filenames, - arg_overrides, - task, - strict, - suffix, - num_shards, - state, - ) - return ensemble, args - - -def get_maybe_sharded_checkpoint_filename( - filename: str, suffix: str, shard_idx: int, num_shards: int -) -> str: - orig_filename = filename - filename = filename.replace(".pt", suffix + ".pt") - fsdp_filename = filename[:-3] + f"-shard{shard_idx}.pt" - model_parallel_filename = orig_filename[:-3] + f"_part{shard_idx}.pt" - if PathManager.exists(fsdp_filename): - return fsdp_filename - elif num_shards > 1: - return model_parallel_filename - else: - return filename - - -def load_model_ensemble_and_task( - filenames, - arg_overrides: Optional[Dict[str, Any]] = None, - task=None, - strict=True, - suffix="", - num_shards=1, - state=None, -): - assert state is None or len(filenames) == 1 - - from fairseq import tasks - - assert not ( - strict and num_shards > 1 - ), "Cannot load state dict with strict=True and checkpoint shards > 1" - ensemble = [] - cfg = None - for filename in filenames: - orig_filename = filename - model_shard_state = {"shard_weights": [], "shard_metadata": []} - assert num_shards > 0 - st = time.time() - for shard_idx in range(num_shards): - filename = get_maybe_sharded_checkpoint_filename( - orig_filename, suffix, shard_idx, num_shards - ) - - if not PathManager.exists(filename): - raise IOError("Model file not found: {}".format(filename)) - if state is None: - state = load_checkpoint_to_cpu(filename, arg_overrides) - if "args" in state and state["args"] is not None: - cfg = convert_namespace_to_omegaconf(state["args"]) - elif "cfg" in state and state["cfg"] is not None: - cfg = state["cfg"] - else: - raise RuntimeError( - f"Neither args nor cfg exist in state keys = {state.keys()}" - ) - - if task is None: - task = tasks.setup_task(cfg.task) - - if "task_state" in state: - task.load_state_dict(state["task_state"]) - - if "fsdp_metadata" in state and num_shards > 1: - model_shard_state["shard_weights"].append(state["model"]) - model_shard_state["shard_metadata"].append(state["fsdp_metadata"]) - # check FSDP import before the code goes too far - if not has_FSDP: - raise ImportError( - "Cannot find FullyShardedDataParallel. " - "Please install fairscale with: pip install fairscale" - ) - if shard_idx == num_shards - 1: - consolidated_model_state = FSDP.consolidate_shard_weights( - shard_weights=model_shard_state["shard_weights"], - shard_metadata=model_shard_state["shard_metadata"], - ) - model = task.build_model(cfg.model) - if ( - "optimizer_history" in state - and len(state["optimizer_history"]) > 0 - and "num_updates" in state["optimizer_history"][-1] - ): - model.set_num_updates( - state["optimizer_history"][-1]["num_updates"] - ) - model.load_state_dict( - consolidated_model_state, strict=strict, model_cfg=cfg.model - ) - else: - # model parallel checkpoint or unsharded checkpoint - # support old external tasks - - argspec = inspect.getfullargspec(task.build_model) - if "from_checkpoint" in argspec.args: - model = task.build_model(cfg.model, from_checkpoint=True) - else: - model = task.build_model(cfg.model) - if ( - "optimizer_history" in state - and len(state["optimizer_history"]) > 0 - and "num_updates" in state["optimizer_history"][-1] - ): - model.set_num_updates(state["optimizer_history"][-1]["num_updates"]) - model.load_state_dict( - state["model"], strict=strict, model_cfg=cfg.model - ) - - # reset state so it gets loaded for the next model in ensemble - state = None - if shard_idx % 10 == 0 and shard_idx > 0: - elapsed = time.time() - st - logger.info( - f"Loaded {shard_idx} shards in {elapsed:.2f}s, {elapsed / (shard_idx+1):.2f}s/shard" - ) - - # build model for ensemble - ensemble.append(model) - return ensemble, cfg, task - - -def load_model_ensemble_and_task_from_hf_hub( - model_id, - cache_dir: Optional[str] = None, - arg_overrides: Optional[Dict[str, Any]] = None, - **kwargs: Any, -): - try: - from huggingface_hub import snapshot_download - except ImportError: - raise ImportError( - "You need to install huggingface_hub to use `load_from_hf_hub`. " - "See https://pypi.org/project/huggingface-hub/ for installation." - ) - - library_name = "fairseq" - cache_dir = cache_dir or (Path.home() / ".cache" / library_name).as_posix() - cache_dir = snapshot_download( - model_id, cache_dir=cache_dir, library_name=library_name, **kwargs - ) - - _arg_overrides = arg_overrides or {} - _arg_overrides["data"] = cache_dir - return load_model_ensemble_and_task( - [p.as_posix() for p in Path(cache_dir).glob("*.pt")], - arg_overrides=_arg_overrides, - ) - - -def checkpoint_paths(path, pattern=r"checkpoint(\d+)\.pt", keep_match=False): - """Retrieves all checkpoints found in `path` directory. - - Checkpoints are identified by matching filename to the specified pattern. If - the pattern contains groups, the result will be sorted by the first group in - descending order. - """ - pt_regexp = re.compile(pattern) - files = PathManager.ls(path) - - entries = [] - for i, f in enumerate(files): - m = pt_regexp.fullmatch(f) - if m is not None: - idx = float(m.group(1)) if len(m.groups()) > 0 else i - entries.append((idx, m.group(0))) - if keep_match: - return [(os.path.join(path, x[1]), x[0]) for x in sorted(entries, reverse=True)] - else: - return [os.path.join(path, x[1]) for x in sorted(entries, reverse=True)] - - -def torch_persistent_save(obj, filename, async_write: bool = False): - if async_write: - with PathManager.opena(filename, "wb") as f: - _torch_persistent_save(obj, f) - else: - if PathManager.supports_rename(filename): - # do atomic save - with PathManager.open(filename + ".tmp", "wb") as f: - _torch_persistent_save(obj, f) - PathManager.rename(filename + ".tmp", filename) - else: - # fallback to non-atomic save - with PathManager.open(filename, "wb") as f: - _torch_persistent_save(obj, f) - - -def _torch_persistent_save(obj, f): - if isinstance(f, str): - with PathManager.open(f, "wb") as h: - torch_persistent_save(obj, h) - return - for i in range(3): - try: - return torch.save(obj, f) - except Exception: - if i == 2: - logger.error(traceback.format_exc()) - raise - - -def _upgrade_state_dict(state): - """Helper for upgrading old model checkpoints.""" - - # add optimizer_history - if "optimizer_history" not in state: - state["optimizer_history"] = [ - {"criterion_name": "CrossEntropyCriterion", "best_loss": state["best_loss"]} - ] - state["last_optimizer_state"] = state["optimizer"] - del state["optimizer"] - del state["best_loss"] - # move extra_state into sub-dictionary - if "epoch" in state and "extra_state" not in state: - state["extra_state"] = { - "epoch": state["epoch"], - "batch_offset": state["batch_offset"], - "val_loss": state["val_loss"], - } - del state["epoch"] - del state["batch_offset"] - del state["val_loss"] - # reduce optimizer history's memory usage (only keep the last state) - if "optimizer" in state["optimizer_history"][-1]: - state["last_optimizer_state"] = state["optimizer_history"][-1]["optimizer"] - for optim_hist in state["optimizer_history"]: - del optim_hist["optimizer"] - # record the optimizer class name - if "optimizer_name" not in state["optimizer_history"][-1]: - state["optimizer_history"][-1]["optimizer_name"] = "FairseqNAG" - # move best_loss into lr_scheduler_state - if "lr_scheduler_state" not in state["optimizer_history"][-1]: - state["optimizer_history"][-1]["lr_scheduler_state"] = { - "best": state["optimizer_history"][-1]["best_loss"] - } - del state["optimizer_history"][-1]["best_loss"] - # keep track of number of updates - if "num_updates" not in state["optimizer_history"][-1]: - state["optimizer_history"][-1]["num_updates"] = 0 - # use stateful training data iterator - if "train_iterator" not in state["extra_state"]: - state["extra_state"]["train_iterator"] = { - "epoch": state["extra_state"].get("epoch", 0), - "iterations_in_epoch": state["extra_state"].get("batch_offset", 0), - } - - # backward compatibility, cfg updates - if "args" in state and state["args"] is not None: - # old model checkpoints may not have separate source/target positions - if hasattr(state["args"], "max_positions") and not hasattr( - state["args"], "max_source_positions" - ): - state["args"].max_source_positions = state["args"].max_positions - state["args"].max_target_positions = state["args"].max_positions - # default to translation task - if not hasattr(state["args"], "task"): - state["args"].task = "translation" - # --raw-text and --lazy-load are deprecated - if getattr(state["args"], "raw_text", False): - state["args"].dataset_impl = "raw" - elif getattr(state["args"], "lazy_load", False): - state["args"].dataset_impl = "lazy" - # epochs start at 1 - if state["extra_state"]["train_iterator"] is not None: - state["extra_state"]["train_iterator"]["epoch"] = max( - state["extra_state"]["train_iterator"].get("epoch", 1), 1 - ) - # --remove-bpe ==> --postprocess - if hasattr(state["args"], "remove_bpe"): - state["args"].post_process = state["args"].remove_bpe - # --min-lr ==> --stop-min-lr - if hasattr(state["args"], "min_lr"): - state["args"].stop_min_lr = state["args"].min_lr - del state["args"].min_lr - # binary_cross_entropy / kd_binary_cross_entropy => wav2vec criterion - if hasattr(state["args"], "criterion") and state["args"].criterion in [ - "binary_cross_entropy", - "kd_binary_cross_entropy", - ]: - state["args"].criterion = "wav2vec" - # remove log_keys if it's None (criteria will supply a default value of []) - if hasattr(state["args"], "log_keys") and state["args"].log_keys is None: - delattr(state["args"], "log_keys") - # speech_pretraining => audio pretraining - if ( - hasattr(state["args"], "task") - and state["args"].task == "speech_pretraining" - ): - state["args"].task = "audio_pretraining" - # audio_cpc => wav2vec - if hasattr(state["args"], "arch") and state["args"].arch == "audio_cpc": - state["args"].arch = "wav2vec" - # convert legacy float learning rate to List[float] - if hasattr(state["args"], "lr") and isinstance(state["args"].lr, float): - state["args"].lr = [state["args"].lr] - # convert task data arg to a string instead of List[string] - if ( - hasattr(state["args"], "data") - and isinstance(state["args"].data, list) - and len(state["args"].data) > 0 - ): - state["args"].data = state["args"].data[0] - - state["cfg"] = convert_namespace_to_omegaconf(state["args"]) - - if "cfg" in state and state["cfg"] is not None: - cfg = state["cfg"] - with open_dict(cfg): - # any upgrades for Hydra-based configs - if ( - "task" in cfg - and "eval_wer_config" in cfg.task - and isinstance(cfg.task.eval_wer_config.print_alignment, bool) - ): - cfg.task.eval_wer_config.print_alignment = "hard" - if "generation" in cfg and isinstance(cfg.generation.print_alignment, bool): - cfg.generation.print_alignment = ( - "hard" if cfg.generation.print_alignment else None - ) - if ( - "model" in cfg - and "w2v_args" in cfg.model - and cfg.model.w2v_args is not None - and ( - hasattr(cfg.model.w2v_args, "task") or "task" in cfg.model.w2v_args - ) - and hasattr(cfg.model.w2v_args.task, "eval_wer_config") - and cfg.model.w2v_args.task.eval_wer_config is not None - and isinstance( - cfg.model.w2v_args.task.eval_wer_config.print_alignment, bool - ) - ): - cfg.model.w2v_args.task.eval_wer_config.print_alignment = "hard" - - return state - - -def prune_state_dict(state_dict, model_cfg: Optional[DictConfig]): - """Prune the given state_dict if desired for LayerDrop - (https://arxiv.org/abs/1909.11556). - - Training with LayerDrop allows models to be robust to pruning at inference - time. This function prunes state_dict to allow smaller models to be loaded - from a larger model and re-maps the existing state_dict for this to occur. - - It's called by functions that load models from checkpoints and does not - need to be called directly. - """ - arch = None - if model_cfg is not None: - arch = ( - model_cfg._name - if isinstance(model_cfg, DictConfig) - else getattr(model_cfg, "arch", None) - ) - - if not model_cfg or arch is None or arch == "ptt_transformer": - # args should not be none, but don't crash if it is. - return state_dict - - encoder_layers_to_keep = getattr(model_cfg, "encoder_layers_to_keep", None) - decoder_layers_to_keep = getattr(model_cfg, "decoder_layers_to_keep", None) - - if not encoder_layers_to_keep and not decoder_layers_to_keep: - return state_dict - - # apply pruning - logger.info( - "Pruning model to specified layer configuration - this works best if the model was trained with LayerDrop" - ) - - def create_pruning_pass(layers_to_keep, layer_name): - keep_layers = sorted( - int(layer_string) for layer_string in layers_to_keep.split(",") - ) - mapping_dict = {} - for i in range(len(keep_layers)): - mapping_dict[str(keep_layers[i])] = str(i) - - regex = re.compile(r"^{layer}.*\.layers\.(\d+)".format(layer=layer_name)) - return {"substitution_regex": regex, "mapping_dict": mapping_dict} - - pruning_passes = [] - if encoder_layers_to_keep: - pruning_passes.append(create_pruning_pass(encoder_layers_to_keep, "encoder")) - if decoder_layers_to_keep: - pruning_passes.append(create_pruning_pass(decoder_layers_to_keep, "decoder")) - - new_state_dict = {} - for layer_name in state_dict.keys(): - match = re.search(r"\.layers\.(\d+)\.", layer_name) - # if layer has no number in it, it is a supporting layer, such as an - # embedding - if not match: - new_state_dict[layer_name] = state_dict[layer_name] - continue - - # otherwise, layer should be pruned. - original_layer_number = match.group(1) - # figure out which mapping dict to replace from - for pruning_pass in pruning_passes: - if original_layer_number in pruning_pass["mapping_dict"] and pruning_pass[ - "substitution_regex" - ].search(layer_name): - new_layer_number = pruning_pass["mapping_dict"][original_layer_number] - substitution_match = pruning_pass["substitution_regex"].search( - layer_name - ) - new_state_key = ( - layer_name[: substitution_match.start(1)] - + new_layer_number - + layer_name[substitution_match.end(1) :] - ) - new_state_dict[new_state_key] = state_dict[layer_name] - - # Since layers are now pruned, *_layers_to_keep are no longer needed. - # This is more of "It would make it work fix" rather than a proper fix. - if isinstance(model_cfg, DictConfig): - context = open_dict(model_cfg) - else: - context = contextlib.ExitStack() - with context: - if hasattr(model_cfg, "encoder_layers_to_keep"): - model_cfg.encoder_layers_to_keep = None - if hasattr(model_cfg, "decoder_layers_to_keep"): - model_cfg.decoder_layers_to_keep = None - - return new_state_dict - - -def load_pretrained_component_from_model( - component: Union[FairseqEncoder, FairseqDecoder], - checkpoint: str, - strict: bool = True, -): - """ - Load a pretrained FairseqEncoder or FairseqDecoder from checkpoint into the - provided `component` object. If state_dict fails to load, there may be a - mismatch in the architecture of the corresponding `component` found in the - `checkpoint` file. - """ - if not PathManager.exists(checkpoint): - raise IOError("Model file not found: {}".format(checkpoint)) - state = load_checkpoint_to_cpu(checkpoint) - if isinstance(component, FairseqEncoder): - component_type = "encoder" - elif isinstance(component, FairseqDecoder): - component_type = "decoder" - else: - raise ValueError( - "component to load must be either a FairseqEncoder or " - "FairseqDecoder. Loading other component types are not supported." - ) - component_state_dict = OrderedDict() - for key in state["model"].keys(): - if key.startswith(component_type): - # encoder.input_layers.0.0.weight --> input_layers.0.0.weight - component_subkey = key[len(component_type) + 1 :] - component_state_dict[component_subkey] = state["model"][key] - component.load_state_dict(component_state_dict, strict=strict) - return component - - -def verify_checkpoint_directory(save_dir: str) -> None: - if not os.path.exists(save_dir): - os.makedirs(save_dir, exist_ok=True) - temp_file_path = os.path.join(save_dir, "dummy") - try: - with open(temp_file_path, "w"): - pass - except OSError as e: - logger.warning( - "Unable to access checkpoint save directory: {}".format(save_dir) - ) - raise e - else: - os.remove(temp_file_path) - - -def save_ema_as_checkpoint(src_path, dst_path): - state = load_ema_from_checkpoint(src_path) - torch_persistent_save(state, dst_path) - - -def load_ema_from_checkpoint(fpath): - """Loads exponential moving averaged (EMA) checkpoint from input and - returns a model with ema weights. - - Args: - fpath: A string path of checkpoint to load from. - - Returns: - A dict of string keys mapping to various values. The 'model' key - from the returned dict should correspond to an OrderedDict mapping - string parameter names to torch Tensors. - """ - params_dict = collections.OrderedDict() - new_state = None - - with PathManager.open(fpath, "rb") as f: - new_state = torch.load( - f, - map_location=( - lambda s, _: torch.serialization.default_restore_location(s, "cpu") - ), - ) - - # EMA model is stored in a separate "extra state" - model_params = new_state["extra_state"]["ema"] - - for key in list(model_params.keys()): - p = model_params[key] - if isinstance(p, torch.HalfTensor): - p = p.float() - if key not in params_dict: - params_dict[key] = p.clone() - # NOTE: clone() is needed in case of p is a shared parameter - else: - raise ValueError("Key {} is repeated in EMA model params.".format(key)) - - if len(params_dict) == 0: - raise ValueError( - f"Input checkpoint path '{fpath}' does not contain " - "ema model weights, is this model trained with EMA?" - ) - - new_state["model"] = params_dict - return new_state diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/clib/cuda/ngram_repeat_block_cuda.cpp b/audio/safeear/safeear_repo/fairseq_ours/fairseq/clib/cuda/ngram_repeat_block_cuda.cpp deleted file mode 100644 index 707219105a17a691e43de1296a72bbaffa0c7fe9..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/clib/cuda/ngram_repeat_block_cuda.cpp +++ /dev/null @@ -1,55 +0,0 @@ -/* -Copyright (c) Microsoft Corporation. -Licensed under the MIT License. -*/ - -#include <torch/extension.h> -#include <vector> - -/* -CPP Binding for CUDA OP -*/ - -// CUDA forward declarations -torch::Tensor ngram_repeat_block_cuda_forward( - torch::Tensor tokens, - torch::Tensor lprobs, - int bsz, - int step, - int beam_size, - int no_repeat_ngram_size); - -#define CHECK_CUDA(x) \ - TORCH_CHECK(x.type().is_cuda(), #x " must be a CUDA tensor") -#define CHECK_CONTIGUOUS(x) \ - TORCH_CHECK(x.is_contiguous(), #x " must be contiguous") -#define CHECK_INPUT(x) \ - CHECK_CUDA(x); \ - CHECK_CONTIGUOUS(x) - -// Input check and call to CUDA OP -// Backward method not required -torch::Tensor ngram_repeat_block_forward( - torch::Tensor tokens, - torch::Tensor lprobs, - int bsz, - int step, - int beam_size, - int no_repeat_ngram_size) { - CHECK_INPUT(tokens); - CHECK_INPUT(lprobs); - assert(bsz > 0); - assert(step >= 0); - assert(beam_size > 0); - assert(no_repeat_ngram_size > 0); - - return ngram_repeat_block_cuda_forward( - tokens, lprobs, bsz, step, beam_size, no_repeat_ngram_size); -} - -PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { - m.def( - "forward", - &ngram_repeat_block_forward, - "No Repeat Ngram Block forward (CUDA)"); -} diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/clib/cuda/ngram_repeat_block_cuda_kernel.cu b/audio/safeear/safeear_repo/fairseq_ours/fairseq/clib/cuda/ngram_repeat_block_cuda_kernel.cu deleted file mode 100644 index bd6106cba0672c3ff29c925b0f5cea557ab3eced..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/clib/cuda/ngram_repeat_block_cuda_kernel.cu +++ /dev/null @@ -1,82 +0,0 @@ -/* -Copyright (c) Microsoft Corporation. -Licensed under the MIT License. -*/ - -/* -Kernel implementation for blocking repeated n-grams. -*/ - -#include <cuda.h> -#include <cuda_runtime.h> -#include <math.h> -#include <torch/extension.h> -#include <vector> - -// Ban repeated ngrams of length = 'no_repeat_ngram_size' -__global__ void banRepeatedTokens( - long* __restrict__ tokens, - float* __restrict__ lprobs, - int max_predict_len, - int vocab_size, - int no_repeat_ngram_size) { - auto row = blockIdx.x; - auto col = threadIdx.x; - auto start = row * (max_predict_len) + col; - // Each thread compares ngram starting from - // thread index with final ngram starting from - // step - no_repeat_ngram_size +2 - auto check_start_pos = blockDim.x; - auto lprob_start = row * vocab_size; - bool is_banned = true; - extern __shared__ long tokens_shm[]; - tokens_shm[col] = tokens[start]; - if (col == blockDim.x - 1) { - for (int i = 1; i < no_repeat_ngram_size; i++) { - if (col + i < max_predict_len) { - tokens_shm[col + i] = tokens[start + i]; - } - } - } - __syncthreads(); - - for (int k = 0; k < no_repeat_ngram_size - 1; k++) { - if (tokens_shm[col + k] != tokens_shm[check_start_pos + k]) { - is_banned = false; - } - } - if (is_banned == true) { - auto token_to_be_banned = tokens_shm[col + no_repeat_ngram_size - 1]; - lprobs[lprob_start + token_to_be_banned] = -INFINITY; - } -} - -// Allocate blocks and threads based on -// batch size and sequence length and launch -// kernel -torch::Tensor ngram_repeat_block_cuda_forward( - const torch::Tensor tokens, - torch::Tensor lprobs, - int bsz, - int step, - int beam_size, - int no_repeat_ngram_size) { - int threads = step - no_repeat_ngram_size + 2; - if (threads <= 0) - return lprobs; - int max_predict_len = tokens.size(1); - int vocab_size = lprobs.size(1); - auto token_ptr = tokens.data_ptr<long>(); - auto lprob_ptr = lprobs.data_ptr<float>(); - int blocks = bsz * beam_size; - int shared_mem_size = (step + 1) * sizeof(long); - - // Launching N blocks where N is number of samples in a batch (beams*bsz) - // Launching T threads where T is number of previous ngrams in a sample - // Allocating shared mem per block for fastser access of input tokens since - // each token will be accessed N times to compare with current Ngram where - // N is Ngram size. - banRepeatedTokens<<<blocks, threads, shared_mem_size>>>( - token_ptr, lprob_ptr, max_predict_len, vocab_size, no_repeat_ngram_size); - return lprobs; -} diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/clib/libbase/balanced_assignment.cpp b/audio/safeear/safeear_repo/fairseq_ours/fairseq/clib/libbase/balanced_assignment.cpp deleted file mode 100644 index 1a5a1061f3892be5a17e49192f744c39e0d395e8..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/clib/libbase/balanced_assignment.cpp +++ /dev/null @@ -1,109 +0,0 @@ -/** - * Copyright 2017-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the license found in the - * LICENSE file in the root directory of this source tree. - */ - -/* -C++ code for solving the linear assignment problem. -Based on the Auction Algorithm from -https://dspace.mit.edu/bitstream/handle/1721.1/3265/P-2108-26912652.pdf and the -implementation from: https://github.com/bkj/auction-lap Adapted to be more -efficient when each worker is looking for k jobs instead of 1. -*/ -#include <torch/extension.h> -#include <iostream> -using namespace torch::indexing; -torch::Tensor balanced_assignment(torch::Tensor job_and_worker_to_score) { - int max_iterations = 100; - torch::Tensor epsilon = - (job_and_worker_to_score.max() - job_and_worker_to_score.min()) / 50; - epsilon.clamp_min_(1e-04); - torch::Tensor worker_and_job_to_score = - job_and_worker_to_score.detach().transpose(0, 1).contiguous(); - int num_workers = worker_and_job_to_score.size(0); - int num_jobs = worker_and_job_to_score.size(1); - auto device = worker_and_job_to_score.device(); - int jobs_per_worker = num_jobs / num_workers; - torch::Tensor value = worker_and_job_to_score.clone(); - int counter = 0; - torch::Tensor max_value = worker_and_job_to_score.max(); - - torch::Tensor bid_indices; - torch::Tensor cost = worker_and_job_to_score.new_zeros({1, num_jobs}); - torch::Tensor bids = - worker_and_job_to_score.new_empty({num_workers, num_jobs}); - torch::Tensor bid_increments = - worker_and_job_to_score.new_empty({num_workers, jobs_per_worker}); - torch::Tensor top_values = - worker_and_job_to_score.new_empty({num_workers, jobs_per_worker + 1}); - torch::Tensor high_bids = worker_and_job_to_score.new_empty({num_jobs}); - - torch::Tensor top_index = top_values.to(torch::kLong); - torch::Tensor high_bidders = top_index.new_empty({num_jobs}); - torch::Tensor have_bids = high_bidders.to(torch::kBool); - torch::Tensor jobs_indices = - torch::arange({num_jobs}, torch::dtype(torch::kLong).device(device)); - torch::Tensor true_tensor = - torch::ones({1}, torch::dtype(torch::kBool).device(device)); - - while (true) { - bids.zero_(); - torch::topk_out(top_values, top_index, value, jobs_per_worker + 1, 1); - - // Each worker bids the difference in value between that job and the k+1th - // job - torch::sub_out( - bid_increments, - top_values.index({Slice(None, None), Slice(0, jobs_per_worker)}), - top_values.index({Slice(None, None), jobs_per_worker}).unsqueeze(1)); - - bid_increments.add_(epsilon); - bids.scatter_( - 1, - top_index.index({Slice(None, None), Slice(0, jobs_per_worker)}), - bid_increments); - - if (counter < max_iterations && counter > 0) { - // Put in a minimal bid to retain items from the last round if no-one else - // bids for them this round - bids.view(-1).index_put_({bid_indices}, epsilon); - } - - // Find the highest bidding worker per job - torch::max_out(high_bids, high_bidders, bids, 0); - torch::gt_out(have_bids, high_bids, 0); - - if (have_bids.all().item<bool>()) { - // All jobs were bid for - break; - } - - // Make popular items more expensive - cost.add_(high_bids); - torch::sub_out(value, worker_and_job_to_score, cost); - - bid_indices = ((high_bidders * num_jobs) + jobs_indices).index({have_bids}); - - if (counter < max_iterations) { - // Make sure that this item will be in the winning worker's top-k next - // time. - value.view(-1).index_put_({bid_indices}, max_value); - } else { - // Suboptimal approximation that converges quickly from current solution - value.view(-1).index_put_( - {bid_indices}, worker_and_job_to_score.view(-1).index({bid_indices})); - } - - counter += 1; - } - - return top_index.index({Slice(None, None), Slice(0, jobs_per_worker)}) - .reshape(-1); -} - -PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { - m.def("balanced_assignment", &balanced_assignment, "Balanced Assignment"); -} diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/clib/libbleu/libbleu.cpp b/audio/safeear/safeear_repo/fairseq_ours/fairseq/clib/libbleu/libbleu.cpp deleted file mode 100644 index 939d9e1174e398fa48c840009b592c753a67939a..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/clib/libbleu/libbleu.cpp +++ /dev/null @@ -1,157 +0,0 @@ -/** - * Copyright 2017-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the license found in the - * LICENSE file in the root directory of this source tree. - */ - -#include <array> -#include <cstdio> -#include <cstring> -#include <map> - -// NOLINTNEXTLINE -typedef struct { - size_t reflen; - size_t predlen; - size_t match1; - size_t count1; - size_t match2; - size_t count2; - size_t match3; - size_t count3; - size_t match4; - size_t count4; -} bleu_stat; - -// left trim (remove pad) -void bleu_ltrim(size_t* len, int** sent, int pad) { - size_t start = 0; - while (start < *len) { - if (*(*sent + start) != pad) { - break; - } - start++; - } - *sent += start; - *len -= start; -} - -// right trim remove (eos) -void bleu_rtrim(size_t* len, int** sent, int pad, int eos) { - size_t end = *len - 1; - while (end > 0) { - if (*(*sent + end) != eos && *(*sent + end) != pad) { - break; - } - end--; - } - *len = end + 1; -} - -// left and right trim -void bleu_trim(size_t* len, int** sent, int pad, int eos) { - bleu_ltrim(len, sent, pad); - bleu_rtrim(len, sent, pad, eos); -} - -size_t bleu_hash(int len, int* data) { - size_t h = 14695981039346656037ul; - size_t prime = 0x100000001b3; - char* b = (char*)data; - size_t blen = sizeof(int) * len; - - while (blen-- > 0) { - h ^= *b++; - h *= prime; - } - - return h; -} - -void bleu_addngram( - size_t* ntotal, - size_t* nmatch, - size_t n, - size_t reflen, - int* ref, - size_t predlen, - int* pred) { - if (predlen < n) { - return; - } - - predlen = predlen - n + 1; - (*ntotal) += predlen; - - if (reflen < n) { - return; - } - - reflen = reflen - n + 1; - - std::map<size_t, size_t> count; - while (predlen > 0) { - size_t w = bleu_hash(n, pred++); - count[w]++; - predlen--; - } - - while (reflen > 0) { - size_t w = bleu_hash(n, ref++); - if (count[w] > 0) { - (*nmatch)++; - count[w] -= 1; - } - reflen--; - } -} - -extern "C" { - -#ifdef _WIN64 -__declspec(dllexport) -#endif - void bleu_zero_init(bleu_stat* stat) { - std::memset(stat, 0, sizeof(bleu_stat)); -} - -#ifdef _WIN64 -__declspec(dllexport) -#endif - void bleu_one_init(bleu_stat* stat) { - bleu_zero_init(stat); - stat->count1 = 0; - stat->count2 = 1; - stat->count3 = 1; - stat->count4 = 1; - stat->match1 = 0; - stat->match2 = 1; - stat->match3 = 1; - stat->match4 = 1; -} - -#ifdef _WIN64 -__declspec(dllexport) -#endif - void bleu_add( - bleu_stat* stat, - size_t reflen, - int* ref, - size_t predlen, - int* pred, - int pad, - int eos) { - - bleu_trim(&reflen, &ref, pad, eos); - bleu_trim(&predlen, &pred, pad, eos); - stat->reflen += reflen; - stat->predlen += predlen; - - bleu_addngram(&stat->count1, &stat->match1, 1, reflen, ref, predlen, pred); - bleu_addngram(&stat->count2, &stat->match2, 2, reflen, ref, predlen, pred); - bleu_addngram(&stat->count3, &stat->match3, 3, reflen, ref, predlen, pred); - bleu_addngram(&stat->count4, &stat->match4, 4, reflen, ref, predlen, pred); -} -} diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/clib/libbleu/module.cpp b/audio/safeear/safeear_repo/fairseq_ours/fairseq/clib/libbleu/module.cpp deleted file mode 100644 index 35288b3177185670135f7bdc1f1589c5bb992304..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/clib/libbleu/module.cpp +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright 2017-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the license found in the - * LICENSE file in the root directory of this source tree. - */ - -#include <Python.h> - -static PyMethodDef method_def[] = {{NULL, NULL, 0, NULL}}; // NOLINT - -static struct PyModuleDef module_def = { - PyModuleDef_HEAD_INIT, - "libbleu", /* name of module */ - // NOLINTNEXTLINE - NULL, /* module documentation, may be NULL */ - -1, /* size of per-interpreter state of the module, - or -1 if the module keeps state in global variables. */ - method_def}; // NOLINT - -#if PY_MAJOR_VERSION == 2 -PyMODINIT_FUNC init_libbleu() -#else -PyMODINIT_FUNC PyInit_libbleu() -#endif -{ - PyObject* m = PyModule_Create(&module_def); - if (!m) { - return NULL; - } - return m; -} diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/clib/libnat/edit_dist.cpp b/audio/safeear/safeear_repo/fairseq_ours/fairseq/clib/libnat/edit_dist.cpp deleted file mode 100644 index 9ffb60569d74d2868ed8113b7c787ef870e9da20..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/clib/libnat/edit_dist.cpp +++ /dev/null @@ -1,231 +0,0 @@ -/** - * Copyright 2017-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the license found in the - * LICENSE file in the root directory of this source tree. - */ - -#include <pybind11/detail/common.h> -#include <pybind11/pybind11.h> -#include <torch/torch.h> // @manual=//caffe2:torch_extension -#include <algorithm> -#include <cstdint> -#include <iosfwd> -#include <memory> -#include <new> -#include <string> -#include <utility> -#include <vector> - -using namespace ::std; - -vector<vector<uint32_t>> edit_distance2_with_dp( - vector<uint32_t>& x, - vector<uint32_t>& y) { - uint32_t lx = x.size(); - uint32_t ly = y.size(); - vector<vector<uint32_t>> d(lx + 1, vector<uint32_t>(ly + 1)); - for (uint32_t i = 0; i < lx + 1; i++) { - d[i][0] = i; - } - for (uint32_t j = 0; j < ly + 1; j++) { - d[0][j] = j; - } - for (uint32_t i = 1; i < lx + 1; i++) { - for (uint32_t j = 1; j < ly + 1; j++) { - d[i][j] = - min(min(d[i - 1][j], d[i][j - 1]) + 1, - d[i - 1][j - 1] + 2 * (x.at(i - 1) == y.at(j - 1) ? 0 : 1)); - } - } - return d; -} - -vector<vector<uint32_t>> edit_distance2_backtracking( - vector<vector<uint32_t>>& d, - vector<uint32_t>& x, - vector<uint32_t>& y, - uint32_t terminal_symbol) { - vector<uint32_t> seq; - vector<vector<uint32_t>> edit_seqs(x.size() + 2, vector<uint32_t>()); - /* - edit_seqs: - 0~x.size() cell is the insertion sequences - last cell is the delete sequence - */ - - if (x.size() == 0) { - edit_seqs.at(0) = y; - return edit_seqs; - } - - uint32_t i = d.size() - 1; - uint32_t j = d.at(0).size() - 1; - - while ((i >= 0) && (j >= 0)) { - if ((i == 0) && (j == 0)) { - break; - } - - if ((j > 0) && (d.at(i).at(j - 1) < d.at(i).at(j))) { - seq.push_back(1); // insert - seq.push_back(y.at(j - 1)); - j--; - } else if ((i > 0) && (d.at(i - 1).at(j) < d.at(i).at(j))) { - seq.push_back(2); // delete - seq.push_back(x.at(i - 1)); - i--; - } else { - seq.push_back(3); // keep - seq.push_back(x.at(i - 1)); - i--; - j--; - } - } - - uint32_t prev_op, op, s, word; - prev_op = 0, s = 0; - for (uint32_t k = 0; k < seq.size() / 2; k++) { - op = seq.at(seq.size() - 2 * k - 2); - word = seq.at(seq.size() - 2 * k - 1); - if (prev_op != 1) { - s++; - } - if (op == 1) // insert - { - edit_seqs.at(s - 1).push_back(word); - } else if (op == 2) // delete - { - edit_seqs.at(x.size() + 1).push_back(1); - } else { - edit_seqs.at(x.size() + 1).push_back(0); - } - - prev_op = op; - } - - for (uint32_t k = 0; k < edit_seqs.size(); k++) { - if (edit_seqs[k].size() == 0) { - edit_seqs[k].push_back(terminal_symbol); - } - } - return edit_seqs; -} - -vector<vector<uint32_t>> edit_distance2_backtracking_with_delete( - vector<vector<uint32_t>>& d, - vector<uint32_t>& x, - vector<uint32_t>& y, - uint32_t terminal_symbol, - uint32_t deletion_symbol) { - vector<uint32_t> seq; - vector<vector<uint32_t>> edit_seqs(x.size() + 1, vector<uint32_t>()); - /* - edit_seqs: - 0~x.size() cell is the insertion sequences - last cell is the delete sequence - */ - - if (x.size() == 0) { - edit_seqs.at(0) = y; - return edit_seqs; - } - - uint32_t i = d.size() - 1; - uint32_t j = d.at(0).size() - 1; - - while ((i >= 0) && (j >= 0)) { - if ((i == 0) && (j == 0)) { - break; - } - - if ((j > 0) && (d.at(i).at(j - 1) < d.at(i).at(j))) { - seq.push_back(1); // insert - seq.push_back(y.at(j - 1)); - j--; - } else if ((i > 0) && (d.at(i - 1).at(j) < d.at(i).at(j))) { - seq.push_back(2); // delete - seq.push_back(x.at(i - 1)); - i--; - } else { - seq.push_back(3); // keep - seq.push_back(x.at(i - 1)); - i--; - j--; - } - } - - uint32_t prev_op, op, s, word; - prev_op = 0, s = 0; - for (uint32_t k = 0; k < seq.size() / 2; k++) { - op = seq.at(seq.size() - 2 * k - 2); - word = seq.at(seq.size() - 2 * k - 1); - if (prev_op != 1) { - s++; - } - if (op == 1) // insert - { - edit_seqs.at(s - 1).push_back(word); - } else if (op == 2) // delete - { - edit_seqs.at(s - 1).push_back(deletion_symbol); - } - - prev_op = op; - } - - for (uint32_t k = 0; k < edit_seqs.size(); k++) { - if (edit_seqs.at(k).size() == 0) { - edit_seqs.at(k).push_back(terminal_symbol); - } - } - return edit_seqs; -} - -vector<uint32_t> compute_ed2( - vector<vector<uint32_t>>& xs, - vector<vector<uint32_t>>& ys) { - vector<uint32_t> distances(xs.size()); - for (uint32_t i = 0; i < xs.size(); i++) { - vector<vector<uint32_t>> d = edit_distance2_with_dp(xs.at(i), ys.at(i)); - distances.at(i) = d.at(xs.at(i).size()).at(ys.at(i).size()); - } - return distances; -} - -vector<vector<vector<uint32_t>>> suggested_ed2_path( - vector<vector<uint32_t>>& xs, - vector<vector<uint32_t>>& ys, - uint32_t terminal_symbol) { - vector<vector<vector<uint32_t>>> seq(xs.size()); - for (uint32_t i = 0; i < xs.size(); i++) { - vector<vector<uint32_t>> d = edit_distance2_with_dp(xs.at(i), ys.at(i)); - seq.at(i) = - edit_distance2_backtracking(d, xs.at(i), ys.at(i), terminal_symbol); - } - return seq; -} - -vector<vector<vector<uint32_t>>> suggested_ed2_path_with_delete( - vector<vector<uint32_t>>& xs, - vector<vector<uint32_t>>& ys, - uint32_t terminal_symbol, - uint32_t deletion_symbol) { - vector<vector<vector<uint32_t>>> seq(xs.size()); - for (uint32_t i = 0; i < xs.size(); i++) { - vector<vector<uint32_t>> d = edit_distance2_with_dp(xs.at(i), ys.at(i)); - seq.at(i) = edit_distance2_backtracking_with_delete( - d, xs.at(i), ys.at(i), terminal_symbol, deletion_symbol); - } - return seq; -} - -PYBIND11_MODULE(libnat, m) { - m.def("compute_ed2", &compute_ed2, "compute_ed2"); - m.def("suggested_ed2_path", &suggested_ed2_path, "suggested_ed2_path"); - m.def( - "suggested_ed2_path_with_delete", - &suggested_ed2_path_with_delete, - "suggested_ed2_path_with_delete"); -} diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/clib/libnat_cuda/binding.cpp b/audio/safeear/safeear_repo/fairseq_ours/fairseq/clib/libnat_cuda/binding.cpp deleted file mode 100644 index ced91c0d0afab9071842911d9876e6360d90284a..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/clib/libnat_cuda/binding.cpp +++ /dev/null @@ -1,67 +0,0 @@ -/** - * Copyright 2017-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the license found in the - * LICENSE file in the root directory of this source tree. - */ - -/* - This code is partially adpoted from - https://github.com/1ytic/pytorch-edit-distance - */ - -#include <torch/types.h> -#include "edit_dist.h" - -#ifndef TORCH_CHECK -#define TORCH_CHECK AT_CHECK -#endif - -#define CHECK_CUDA(x) \ - TORCH_CHECK(x.type().is_cuda(), #x " must be a CUDA tensor") -#define CHECK_CONTIGUOUS(x) \ - TORCH_CHECK(x.is_contiguous(), #x " must be contiguous") -#define CHECK_INPUT(x) \ - CHECK_CUDA(x); \ - CHECK_CONTIGUOUS(x) - -torch::Tensor LevenshteinDistance( - torch::Tensor source, - torch::Tensor target, - torch::Tensor source_length, - torch::Tensor target_length) { - CHECK_INPUT(source); - CHECK_INPUT(target); - CHECK_INPUT(source_length); - CHECK_INPUT(target_length); - return LevenshteinDistanceCuda(source, target, source_length, target_length); -} - -torch::Tensor GenerateDeletionLabel( - torch::Tensor source, - torch::Tensor operations) { - CHECK_INPUT(source); - CHECK_INPUT(operations); - return GenerateDeletionLabelCuda(source, operations); -} - -std::pair<torch::Tensor, torch::Tensor> GenerateInsertionLabel( - torch::Tensor target, - torch::Tensor operations) { - CHECK_INPUT(target); - CHECK_INPUT(operations); - return GenerateInsertionLabelCuda(target, operations); -} - -PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { - m.def("levenshtein_distance", &LevenshteinDistance, "Levenshtein distance"); - m.def( - "generate_deletion_labels", - &GenerateDeletionLabel, - "Generate Deletion Label"); - m.def( - "generate_insertion_labels", - &GenerateInsertionLabel, - "Generate Insertion Label"); -} diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/clib/libnat_cuda/edit_dist.cu b/audio/safeear/safeear_repo/fairseq_ours/fairseq/clib/libnat_cuda/edit_dist.cu deleted file mode 100644 index 96569d46c8475a90f3d51152023a46780464ecbd..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/clib/libnat_cuda/edit_dist.cu +++ /dev/null @@ -1,344 +0,0 @@ -/** - * Copyright 2017-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the license found in the - * LICENSE file in the root directory of this source tree. - */ - -#include "edit_dist.h" - -#include <THC/THC.h> -#include <cuda.h> -#include <cuda_runtime.h> -#include <device_launch_parameters.h> -#include <utility> // std::pair - -template <typename scalar_t> -__global__ void generate_deletion_label_kernel( - const scalar_t* __restrict__ source, - const size_t source_size, - const size_t operation_size, - int* __restrict__ operations, - int* __restrict__ labels) { - const int index = blockIdx.x; - const int offset = index * operation_size; - const int offset_label = index * source_size; - - for (int i = 0; i < source_size; i++) { - labels[offset_label + i] = 0; - } - - int k = 0; - for (int i = 0; i < operation_size; i++) { - if (operations[offset + i] == 0) { - break; - } else if (operations[offset + i] == 1) { - continue; - } else { - labels[offset_label + k] = 3 - operations[offset + i]; - k++; - } - } -} - -template <typename scalar_t> -__global__ void generate_insertion_label_kernel( - const scalar_t* __restrict__ target, - const size_t target_size, - const size_t operation_size, - int* __restrict__ operations, - int* __restrict__ labels, - int* __restrict__ masks) { - const int index = blockIdx.x; - const int offset = index * operation_size; - const int offset_label = index * target_size; - - int k = 0; - int u = 0; - int m = 0; - - for (int i = 0; i < target_size; i++) { - labels[offset_label + i] = 0; - masks[offset_label + i] = 0; - } - - for (int i = 0; i < operation_size - 1; i++) { - if (operations[offset + i] == 0) { - break; - } else if (operations[offset + i] == 2) { - continue; - } else if (operations[offset + i] == 1) { - masks[offset_label + m] = 1; - u++; - m++; - } else { - labels[offset_label + k] = u; - masks[offset_label + m] = 0; - k++; - m++; - u = 0; - } - } -} - -template <typename scalar_t> -__global__ void levenshtein_distance_kernel( - const scalar_t* __restrict__ source, - const scalar_t* __restrict__ target, - const int* __restrict__ source_length, - const int* __restrict__ target_length, - const size_t source_size, - const size_t target_size, - int* __restrict__ operations, - int* __restrict__ errors_curr) { - const int index = blockIdx.x; - const int offset = index * (source_size + target_size); - const int d = index * (source_size + 1) * (target_size + 1); - const int t = target_size + 1; - - auto err_idx = [d, t](int i, int j) { return d + i * t + j; }; - auto opt_idx = [offset](int k) { return offset + k; }; - - const int hyp_len = source_length[index]; - const int ref_len = target_length[index]; - const scalar_t* hyp_begin = source + index * source_size; - const scalar_t* ref_begin = target + index * target_size; - - // dynamic programming - for (int i = 0; i <= hyp_len; i++) { - errors_curr[err_idx(i, 0)] = i; - } - for (int j = 0; j <= ref_len; j++) { - errors_curr[err_idx(0, j)] = j; - } - for (int i = 1; i <= hyp_len; i++) { - for (int j = 1; j <= ref_len; j++) { - errors_curr[err_idx(i, j)] = min( - min(errors_curr[err_idx(i - 1, j)], errors_curr[err_idx(i, j - 1)]) + - 1, - errors_curr[err_idx(i - 1, j - 1)] + - 2 * (*(hyp_begin + i - 1) == *(ref_begin + j - 1) ? 0 : 1)); - } - } - - // back-tracing - int i = hyp_len; - int j = ref_len; - int o = hyp_len + ref_len; - - for (int k = 0; k < source_size + target_size; k++) { - operations[opt_idx(k)] = 0; - } - - while ((i >= 0) && (j >= 0)) { - if ((i == 0) && (j == 0)) { - break; - } - - if ((j > 0) && - (errors_curr[err_idx(i, j - 1)] < errors_curr[err_idx(i, j)])) { - o--; - operations[opt_idx(o)] = 1; - j--; // insertion - } else if ( - (i > 0) && - (errors_curr[err_idx(i - 1, j)] < errors_curr[err_idx(i, j)])) { - o--; - operations[opt_idx(o)] = 2; - i--; // deletion - } else { - o--; - operations[opt_idx(o)] = 3; - i--; - j--; // do nothing - } - } - - // moving to the left - for (int k = 0; k < hyp_len + ref_len; k++) { - if (k + o < hyp_len + ref_len) { - operations[opt_idx(k)] = operations[opt_idx(k + o)]; - } else { - operations[opt_idx(k)] = 0; // padding - } - } -} - -template <typename scalar_t> -__global__ void faster_levenshtein_distance_kernel( - const scalar_t* __restrict__ source, - const scalar_t* __restrict__ target, - const int* __restrict__ source_length, - const int* __restrict__ target_length, - const size_t source_size, - const size_t target_size, - int* __restrict__ operations) { - extern __shared__ short errors[]; - auto errors_curr = errors; - - const int index = blockIdx.x; - const int offset = index * (source_size + target_size); - const int t = target_size + 1; - - auto err_idx = [t](int i, int j) { return i * t + j; }; - auto opt_idx = [offset](int k) { return offset + k; }; - - const int hyp_len = source_length[index]; - const int ref_len = target_length[index]; - const scalar_t* hyp_begin = source + index * source_size; - const scalar_t* ref_begin = target + index * target_size; - - // dynamic programming - for (int i = 0; i <= hyp_len; i++) { - errors_curr[err_idx(i, 0)] = i; - } - for (int j = 0; j <= ref_len; j++) { - errors_curr[err_idx(0, j)] = j; - } - for (int i = 1; i <= hyp_len; i++) { - for (int j = 1; j <= ref_len; j++) { - errors_curr[err_idx(i, j)] = min( - min(errors_curr[err_idx(i - 1, j)], errors_curr[err_idx(i, j - 1)]) + - 1, - errors_curr[err_idx(i - 1, j - 1)] + - 2 * (*(hyp_begin + i - 1) == *(ref_begin + j - 1) ? 0 : 1)); - } - } - - // back-tracing - int i = hyp_len; - int j = ref_len; - int o = hyp_len + ref_len; - - for (int k = 0; k < source_size + target_size; k++) { - operations[opt_idx(k)] = 0; - } - - while ((i >= 0) && (j >= 0)) { - if ((i == 0) && (j == 0)) { - break; - } - - if ((j > 0) && - (errors_curr[err_idx(i, j - 1)] < errors_curr[err_idx(i, j)])) { - o--; - operations[opt_idx(o)] = 1; - j--; // insertion - } else if ( - (i > 0) && - (errors_curr[err_idx(i - 1, j)] < errors_curr[err_idx(i, j)])) { - o--; - operations[opt_idx(o)] = 2; - i--; // deletion - } else { - o--; - operations[opt_idx(o)] = 3; - i--; - j--; // do nothing - } - } - - // moving to the left - for (int k = 0; k < hyp_len + ref_len; k++) { - if (k + o < hyp_len + ref_len) { - operations[opt_idx(k)] = operations[opt_idx(k + o)]; - } else { - operations[opt_idx(k)] = 0; // padding - } - } -} - -torch::Tensor GenerateDeletionLabelCuda( - torch::Tensor source, - torch::Tensor operations) { - const auto batch_size = source.size(0); - at::TensorOptions options(source.device()); - options = options.dtype(at::ScalarType::Int); - auto labels = torch::empty({batch_size, source.size(1)}, options); - auto stream = at::cuda::getCurrentCUDAStream(source.device().index()); - - AT_DISPATCH_ALL_TYPES(source.scalar_type(), "generate_deletion_labels", ([&] { - generate_deletion_label_kernel<scalar_t> - <<<batch_size, 1, 0, stream>>>( - source.data_ptr<scalar_t>(), - source.size(1), - operations.size(1), - operations.data_ptr<int>(), - labels.data_ptr<int>()); - })); - - return labels; -} - -std::pair<torch::Tensor, torch::Tensor> GenerateInsertionLabelCuda( - torch::Tensor target, - torch::Tensor operations) { - const auto batch_size = target.size(0); - at::TensorOptions options(target.device()); - options = options.dtype(at::ScalarType::Int); - auto labels = torch::empty({batch_size, target.size(1)}, options); - auto masks = torch::empty({batch_size, target.size(1)}, options); - auto stream = at::cuda::getCurrentCUDAStream(target.device().index()); - - AT_DISPATCH_ALL_TYPES( - target.scalar_type(), "generate_insertion_labels", ([&] { - generate_insertion_label_kernel<scalar_t><<<batch_size, 1, 0, stream>>>( - target.data_ptr<scalar_t>(), - target.size(1), - operations.size(1), - operations.data_ptr<int>(), - labels.data_ptr<int>(), - masks.data_ptr<int>()); - })); - - return std::make_pair(labels, masks); -} - -torch::Tensor LevenshteinDistanceCuda( - torch::Tensor source, - torch::Tensor target, - torch::Tensor source_length, - torch::Tensor target_length) { - const auto batch_size = source.size(0); - const auto shared_size = - (source.size(1) + 1) * (target.size(1) + 1) * sizeof(short); - - at::TensorOptions options(source.device()); - options = options.dtype(at::ScalarType::Int); - auto operations = - torch::empty({batch_size, source.size(1) + target.size(1)}, options); - auto stream = at::cuda::getCurrentCUDAStream(source.device().index()); - - if (shared_size > 40000) { - auto distances = torch::empty( - {batch_size, (source.size(1) + 1) * (target.size(1) + 1)}, options); - AT_DISPATCH_ALL_TYPES(source.scalar_type(), "levenshtein_distance", ([&] { - levenshtein_distance_kernel<scalar_t> - <<<batch_size, 1, 0, stream>>>( - source.data_ptr<scalar_t>(), - target.data_ptr<scalar_t>(), - source_length.data_ptr<int>(), - target_length.data_ptr<int>(), - source.size(1), - target.size(1), - operations.data_ptr<int>(), - distances.data_ptr<int>()); - })); - } else { - AT_DISPATCH_ALL_TYPES( - source.scalar_type(), "faster_levenshtein_distance", ([&] { - faster_levenshtein_distance_kernel<scalar_t> - <<<batch_size, 1, shared_size, stream>>>( - source.data_ptr<scalar_t>(), - target.data_ptr<scalar_t>(), - source_length.data_ptr<int>(), - target_length.data_ptr<int>(), - source.size(1), - target.size(1), - operations.data_ptr<int>()); - })); - } - - return operations; -} diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/clib/libnat_cuda/edit_dist.h b/audio/safeear/safeear_repo/fairseq_ours/fairseq/clib/libnat_cuda/edit_dist.h deleted file mode 100644 index 5220c52fd80529b90a67ba74e9ca73c668dab099..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/clib/libnat_cuda/edit_dist.h +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Copyright 2017-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the license found in the - * LICENSE file in the root directory of this source tree. - */ - -#pragma once - -#include <torch/extension.h> - -torch::Tensor LevenshteinDistanceCuda( - torch::Tensor source, - torch::Tensor target, - torch::Tensor source_length, - torch::Tensor target_length); - -torch::Tensor GenerateDeletionLabelCuda( - torch::Tensor source, - torch::Tensor operations); - -std::pair<torch::Tensor, torch::Tensor> GenerateInsertionLabelCuda( - torch::Tensor source, - torch::Tensor operations); diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/config/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/config/__init__.py deleted file mode 100644 index 6264236915a7269a4d920ee8213004374dd86a9a..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/config/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/config/config.yaml b/audio/safeear/safeear_repo/fairseq_ours/fairseq/config/config.yaml deleted file mode 100644 index 2ed7168cb7f7473c43d864478c5c6ce51639e030..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/config/config.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# @package _group_ - -hydra: - run: - dir: . - -defaults: - - _self_ - - task: null - - model: null - - criterion: cross_entropy - - optimizer: null - - lr_scheduler: fixed - - bpe: null - - tokenizer: null - - scoring: null - - generation: null - - common_eval: null - - eval_lm: null diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/config/model/transformer_lm/transformer_lm_baevski_gbw.yaml b/audio/safeear/safeear_repo/fairseq_ours/fairseq/config/model/transformer_lm/transformer_lm_baevski_gbw.yaml deleted file mode 100644 index 30b1a4f1e0f5e7f7c2671ff8ec995cc32363f10f..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/config/model/transformer_lm/transformer_lm_baevski_gbw.yaml +++ /dev/null @@ -1,36 +0,0 @@ -# @package _group_ -activation_fn: "relu" -dropout: 0.1 -attention_dropout: 0.1 -activation_dropout: 0.0 -relu_dropout: 0.0 -decoder_embed_dim: 512 -decoder_output_dim: 512 -decoder_input_dim: 512 -decoder_ffn_embed_dim: 4096 -decoder_layers: 12 -decoder_attention_heads: 16 -decoder_normalize_before: true -no_decoder_final_norm: true -adaptive_softmax_cutoff: null -adaptive_softmax_dropout: 0 -adaptive_softmax_factor: 4 -no_token_positional_embeddings: false -share_decoder_input_output_embed: false -character_embeddings: false -character_filters: "[(1, 64), (2, 128), (3, 192), (4, 256), (5, 256), (6, 256), (7, 256)]" -character_embedding_dim: 4 -char_embedder_highway_layers: 2 -adaptive_input: false -adaptive_input_factor: 4 -adaptive_input_cutoff: null -tie_adaptive_weights: false -tie_adaptive_proj: false -decoder_learned_pos: false -decoder_layerdrop: 0 -decoder_layers_to_keep: null -layernorm_embedding: false -no_scale_embedding: false -quant_noise_pq: 0 -quant_noise_pq_block_size: 8 -quant_noise_scalar: 0 diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/config/model/transformer_lm/transformer_lm_baevski_wiki103.yaml b/audio/safeear/safeear_repo/fairseq_ours/fairseq/config/model/transformer_lm/transformer_lm_baevski_wiki103.yaml deleted file mode 100644 index 1154cfa660ee5ce6a272cd1a0049eead1e92c117..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/config/model/transformer_lm/transformer_lm_baevski_wiki103.yaml +++ /dev/null @@ -1,36 +0,0 @@ -# @package _group_ -activation_fn: "relu" -dropout: 0.3 -attention_dropout: 0.1 -activation_dropout: 0.1 -relu_dropout: 0.1 -decoder_embed_dim: 1024 -decoder_output_dim: 1024 -decoder_input_dim: 1024 -decoder_ffn_embed_dim: 4096 -decoder_layers: 16 -decoder_attention_heads: 8 -decoder_normalize_before: true -no_decoder_final_norm: true -adaptive_softmax_cutoff: "20000,60000" -adaptive_softmax_dropout: 0.2 -adaptive_softmax_factor: 4 -no_token_positional_embeddings: false -share_decoder_input_output_embed: false -character_embeddings: false -character_filters: "[(1, 64), (2, 128), (3, 192), (4, 256), (5, 256), (6, 256), (7, 256)]" -character_embedding_dim: 4 -char_embedder_highway_layers: 2 -adaptive_input: true -adaptive_input_factor: 4 -adaptive_input_cutoff: "20000,60000" -tie_adaptive_weights: true -tie_adaptive_proj: true -decoder_learned_pos: false -decoder_layerdrop: 0 -decoder_layers_to_keep: null -layernorm_embedding: false -no_scale_embedding: false -quant_noise_pq: 0 -quant_noise_pq_block_size: 8 -quant_noise_scalar: 0 diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/config/model/transformer_lm/transformer_lm_big.yaml b/audio/safeear/safeear_repo/fairseq_ours/fairseq/config/model/transformer_lm/transformer_lm_big.yaml deleted file mode 100644 index 309575310bfc5d9c5cde31563073bef18abc646e..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/config/model/transformer_lm/transformer_lm_big.yaml +++ /dev/null @@ -1,36 +0,0 @@ -# @package _group_ -activation_fn: "relu" -dropout: 0.1 -attention_dropout: 0.0 -activation_dropout: 0.0 -relu_dropout: 0.0 -decoder_embed_dim: 1024 -decoder_output_dim: 1024 -decoder_input_dim: 1024 -decoder_ffn_embed_dim: 4096 -decoder_layers: 12 -decoder_attention_heads: 16 -decoder_normalize_before: true -no_decoder_final_norm: false -adaptive_softmax_cutoff: null -adaptive_softmax_dropout: 0 -adaptive_softmax_factor: 4 -no_token_positional_embeddings: false -share_decoder_input_output_embed: false -character_embeddings: false -character_filters: "[(1, 64), (2, 128), (3, 192), (4, 256), (5, 256), (6, 256), (7, 256)]" -character_embedding_dim: 4 -char_embedder_highway_layers: 2 -adaptive_input: false -adaptive_input_factor: 4 -adaptive_input_cutoff: null -tie_adaptive_weights: false -tie_adaptive_proj: false -decoder_learned_pos: false -decoder_layerdrop: 0 -decoder_layers_to_keep: null -layernorm_embedding: false -no_scale_embedding: false -quant_noise_pq: 0 -quant_noise_pq_block_size: 8 -quant_noise_scalar: 0 diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/config/model/transformer_lm/transformer_lm_gbw.yaml b/audio/safeear/safeear_repo/fairseq_ours/fairseq/config/model/transformer_lm/transformer_lm_gbw.yaml deleted file mode 100644 index 30b1a4f1e0f5e7f7c2671ff8ec995cc32363f10f..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/config/model/transformer_lm/transformer_lm_gbw.yaml +++ /dev/null @@ -1,36 +0,0 @@ -# @package _group_ -activation_fn: "relu" -dropout: 0.1 -attention_dropout: 0.1 -activation_dropout: 0.0 -relu_dropout: 0.0 -decoder_embed_dim: 512 -decoder_output_dim: 512 -decoder_input_dim: 512 -decoder_ffn_embed_dim: 4096 -decoder_layers: 12 -decoder_attention_heads: 16 -decoder_normalize_before: true -no_decoder_final_norm: true -adaptive_softmax_cutoff: null -adaptive_softmax_dropout: 0 -adaptive_softmax_factor: 4 -no_token_positional_embeddings: false -share_decoder_input_output_embed: false -character_embeddings: false -character_filters: "[(1, 64), (2, 128), (3, 192), (4, 256), (5, 256), (6, 256), (7, 256)]" -character_embedding_dim: 4 -char_embedder_highway_layers: 2 -adaptive_input: false -adaptive_input_factor: 4 -adaptive_input_cutoff: null -tie_adaptive_weights: false -tie_adaptive_proj: false -decoder_learned_pos: false -decoder_layerdrop: 0 -decoder_layers_to_keep: null -layernorm_embedding: false -no_scale_embedding: false -quant_noise_pq: 0 -quant_noise_pq_block_size: 8 -quant_noise_scalar: 0 diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/config/model/transformer_lm/transformer_lm_gpt.yaml b/audio/safeear/safeear_repo/fairseq_ours/fairseq/config/model/transformer_lm/transformer_lm_gpt.yaml deleted file mode 100644 index 2c6cb7be3801115371566932ffc78651c9ac6c0f..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/config/model/transformer_lm/transformer_lm_gpt.yaml +++ /dev/null @@ -1,36 +0,0 @@ -# @package _group_ -activation_fn: "gelu" -dropout: 0.1 -attention_dropout: 0.1 -activation_dropout: 0.0 -relu_dropout: 0.0 -decoder_embed_dim: 768 -decoder_output_dim: 768 -decoder_input_dim: 768 -decoder_ffn_embed_dim: 3072 -decoder_layers: 12 -decoder_attention_heads: 12 -decoder_normalize_before: true -no_decoder_final_norm: false -adaptive_softmax_cutoff: null -adaptive_softmax_dropout: 0 -adaptive_softmax_factor: 4 -no_token_positional_embeddings: false -share_decoder_input_output_embed: false -character_embeddings: false -character_filters: "[(1, 64), (2, 128), (3, 192), (4, 256), (5, 256), (6, 256), (7, 256)]" -character_embedding_dim: 4 -char_embedder_highway_layers: 2 -adaptive_input: false -adaptive_input_factor: 4 -adaptive_input_cutoff: null -tie_adaptive_weights: false -tie_adaptive_proj: false -decoder_learned_pos: false -decoder_layerdrop: 0 -decoder_layers_to_keep: null -layernorm_embedding: false -no_scale_embedding: false -quant_noise_pq: 0 -quant_noise_pq_block_size: 8 -quant_noise_scalar: 0 diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/config/model/transformer_lm/transformer_lm_gpt2_big.yaml b/audio/safeear/safeear_repo/fairseq_ours/fairseq/config/model/transformer_lm/transformer_lm_gpt2_big.yaml deleted file mode 100644 index a08769a1781abdb13302bf57bf1338bcaf68a0ec..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/config/model/transformer_lm/transformer_lm_gpt2_big.yaml +++ /dev/null @@ -1,36 +0,0 @@ -# @package _group_ -activation_fn: "gelu" -dropout: 0.1 -attention_dropout: 0.1 -activation_dropout: 0.0 -relu_dropout: 0.0 -decoder_embed_dim: 1600 -decoder_output_dim: 1600 -decoder_input_dim: 1600 -decoder_ffn_embed_dim: 6400 -decoder_layers: 48 -decoder_attention_heads: 25 -decoder_normalize_before: true -no_decoder_final_norm: false -adaptive_softmax_cutoff: null -adaptive_softmax_dropout: 0 -adaptive_softmax_factor: 4 -no_token_positional_embeddings: false -share_decoder_input_output_embed: false -character_embeddings: false -character_filters: "[(1, 64), (2, 128), (3, 192), (4, 256), (5, 256), (6, 256), (7, 256)]" -character_embedding_dim: 4 -char_embedder_highway_layers: 2 -adaptive_input: false -adaptive_input_factor: 4 -adaptive_input_cutoff: null -tie_adaptive_weights: false -tie_adaptive_proj: false -decoder_learned_pos: false -decoder_layerdrop: 0 -decoder_layers_to_keep: null -layernorm_embedding: false -no_scale_embedding: false -quant_noise_pq: 0 -quant_noise_pq_block_size: 8 -quant_noise_scalar: 0 diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/config/model/transformer_lm/transformer_lm_gpt2_medium.yaml b/audio/safeear/safeear_repo/fairseq_ours/fairseq/config/model/transformer_lm/transformer_lm_gpt2_medium.yaml deleted file mode 100644 index 64261d793c0f1ae091c9bf5c8c77093a07326137..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/config/model/transformer_lm/transformer_lm_gpt2_medium.yaml +++ /dev/null @@ -1,36 +0,0 @@ -# @package _group_ -activation_fn: "gelu" -dropout: 0.1 -attention_dropout: 0.1 -activation_dropout: 0.0 -relu_dropout: 0.0 -decoder_embed_dim: 1280 -decoder_output_dim: 1280 -decoder_input_dim: 1280 -decoder_ffn_embed_dim: 5120 -decoder_layers: 36 -decoder_attention_heads: 20 -decoder_normalize_before: true -no_decoder_final_norm: false -adaptive_softmax_cutoff: null -adaptive_softmax_dropout: 0 -adaptive_softmax_factor: 4 -no_token_positional_embeddings: false -share_decoder_input_output_embed: false -character_embeddings: false -character_filters: "[(1, 64), (2, 128), (3, 192), (4, 256), (5, 256), (6, 256), (7, 256)]" -character_embedding_dim: 4 -char_embedder_highway_layers: 2 -adaptive_input: false -adaptive_input_factor: 4 -adaptive_input_cutoff: null -tie_adaptive_weights: false -tie_adaptive_proj: false -decoder_learned_pos: false -decoder_layerdrop: 0 -decoder_layers_to_keep: null -layernorm_embedding: false -no_scale_embedding: false -quant_noise_pq: 0 -quant_noise_pq_block_size: 8 -quant_noise_scalar: 0 diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/config/model/transformer_lm/transformer_lm_gpt2_small.yaml b/audio/safeear/safeear_repo/fairseq_ours/fairseq/config/model/transformer_lm/transformer_lm_gpt2_small.yaml deleted file mode 100644 index 702e81f466c82edf40433589d389edbe0a7b96db..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/config/model/transformer_lm/transformer_lm_gpt2_small.yaml +++ /dev/null @@ -1,36 +0,0 @@ -# @package _group_ -activation_fn: "gelu" -dropout: 0.1 -attention_dropout: 0.1 -activation_dropout: 0.0 -relu_dropout: 0.0 -decoder_embed_dim: 1024 -decoder_output_dim: 1024 -decoder_input_dim: 1024 -decoder_ffn_embed_dim: 4096 -decoder_layers: 24 -decoder_attention_heads: 16 -decoder_normalize_before: true -no_decoder_final_norm: false -adaptive_softmax_cutoff: null -adaptive_softmax_dropout: 0 -adaptive_softmax_factor: 4 -no_token_positional_embeddings: false -share_decoder_input_output_embed: false -character_embeddings: false -character_filters: "[(1, 64), (2, 128), (3, 192), (4, 256), (5, 256), (6, 256), (7, 256)]" -character_embedding_dim: 4 -char_embedder_highway_layers: 2 -adaptive_input: false -adaptive_input_factor: 4 -adaptive_input_cutoff: null -tie_adaptive_weights: false -tie_adaptive_proj: false -decoder_learned_pos: false -decoder_layerdrop: 0 -decoder_layers_to_keep: null -layernorm_embedding: false -no_scale_embedding: false -quant_noise_pq: 0 -quant_noise_pq_block_size: 8 -quant_noise_scalar: 0 diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/config/model/transformer_lm/transformer_lm_wiki103.yaml b/audio/safeear/safeear_repo/fairseq_ours/fairseq/config/model/transformer_lm/transformer_lm_wiki103.yaml deleted file mode 100644 index 1154cfa660ee5ce6a272cd1a0049eead1e92c117..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/config/model/transformer_lm/transformer_lm_wiki103.yaml +++ /dev/null @@ -1,36 +0,0 @@ -# @package _group_ -activation_fn: "relu" -dropout: 0.3 -attention_dropout: 0.1 -activation_dropout: 0.1 -relu_dropout: 0.1 -decoder_embed_dim: 1024 -decoder_output_dim: 1024 -decoder_input_dim: 1024 -decoder_ffn_embed_dim: 4096 -decoder_layers: 16 -decoder_attention_heads: 8 -decoder_normalize_before: true -no_decoder_final_norm: true -adaptive_softmax_cutoff: "20000,60000" -adaptive_softmax_dropout: 0.2 -adaptive_softmax_factor: 4 -no_token_positional_embeddings: false -share_decoder_input_output_embed: false -character_embeddings: false -character_filters: "[(1, 64), (2, 128), (3, 192), (4, 256), (5, 256), (6, 256), (7, 256)]" -character_embedding_dim: 4 -char_embedder_highway_layers: 2 -adaptive_input: true -adaptive_input_factor: 4 -adaptive_input_cutoff: "20000,60000" -tie_adaptive_weights: true -tie_adaptive_proj: true -decoder_learned_pos: false -decoder_layerdrop: 0 -decoder_layers_to_keep: null -layernorm_embedding: false -no_scale_embedding: false -quant_noise_pq: 0 -quant_noise_pq_block_size: 8 -quant_noise_scalar: 0 diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/config/model/wav2vec/vq_wav2vec_gumbel.yaml b/audio/safeear/safeear_repo/fairseq_ours/fairseq/config/model/wav2vec/vq_wav2vec_gumbel.yaml deleted file mode 100644 index ee1329bf4612d8bb295c6cc3d8bc0a3bcef1777d..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/config/model/wav2vec/vq_wav2vec_gumbel.yaml +++ /dev/null @@ -1,5 +0,0 @@ -# @package _group_ -activation: gelu -vq_type: gumbel -vq_depth: 2 -combine_groups: true diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/config/model/wav2vec2/wav2vec2_base.yaml b/audio/safeear/safeear_repo/fairseq_ours/fairseq/config/model/wav2vec2/wav2vec2_base.yaml deleted file mode 100644 index ce65499b808b9a3821cee4ca87c36e84d09005a1..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/config/model/wav2vec2/wav2vec2_base.yaml +++ /dev/null @@ -1,8 +0,0 @@ -# @package _group_ - -quantize_targets: true -final_dim: 256 -encoder_layerdrop: 0.05 -dropout_input: 0.1 -dropout_features: 0.1 -feature_grad_mult: 0.1 diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/config/model/wav2vec2/wav2vec2_large.yaml b/audio/safeear/safeear_repo/fairseq_ours/fairseq/config/model/wav2vec2/wav2vec2_large.yaml deleted file mode 100644 index 5846f75243f27f201c85bfe6820815c015971275..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/config/model/wav2vec2/wav2vec2_large.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# @package _group_ - -quantize_targets: true -extractor_mode: layer_norm -layer_norm_first: true -final_dim: 768 -latent_temp: [2.0,0.1,0.999995] -encoder_layerdrop: 0.0 -dropout_input: 0.0 -dropout_features: 0.0 -dropout: 0.0 -attention_dropout: 0.0 -conv_bias: true - -encoder_layers: 24 -encoder_embed_dim: 1024 -encoder_ffn_embed_dim: 4096 -encoder_attention_heads: 16 - -feature_grad_mult: 1.0 diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/__init__.py deleted file mode 100644 index 4dbf46a1cb31ce65c4224ae79cbc2d7cf9e4d111..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/__init__.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. -"""isort:skip_file""" - -import importlib -import os - -from fairseq import registry -from fairseq.criterions.fairseq_criterion import ( # noqa - FairseqCriterion, - LegacyFairseqCriterion, -) -from omegaconf import DictConfig - - -( - build_criterion_, - register_criterion, - CRITERION_REGISTRY, - CRITERION_DATACLASS_REGISTRY, -) = registry.setup_registry( - "--criterion", base_class=FairseqCriterion, default="cross_entropy" -) - - -def build_criterion(cfg: DictConfig, task): - return build_criterion_(cfg, task) - - -# automatically import any Python files in the criterions/ directory -for file in sorted(os.listdir(os.path.dirname(__file__))): - if file.endswith(".py") and not file.startswith("_"): - file_name = file[: file.find(".py")] - importlib.import_module("fairseq.criterions." + file_name) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/adaptive_loss.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/adaptive_loss.py deleted file mode 100644 index 6209ceaedb6d8120ad820c11b55c13596447933c..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/adaptive_loss.py +++ /dev/null @@ -1,123 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import math -from dataclasses import dataclass - -import torch.nn.functional as F -from fairseq import metrics, utils -from fairseq.criterions import FairseqCriterion, register_criterion -from fairseq.dataclass import FairseqDataclass -from fairseq.dataclass.constants import DDP_BACKEND_CHOICES -from omegaconf import II - - -@dataclass -class AdaptiveLossConfig(FairseqDataclass): - sentence_avg: bool = II("optimization.sentence_avg") - ddp_backend: DDP_BACKEND_CHOICES = II("distributed_training.ddp_backend") - - -@register_criterion("adaptive_loss", dataclass=AdaptiveLossConfig) -class AdaptiveLoss(FairseqCriterion): - """This is an implementation of the loss function accompanying the adaptive softmax approximation for - graphical processing units (GPU), described in the paper "Efficient softmax approximation for GPUs" - (http://arxiv.org/abs/1609.04309).""" - - def __init__(self, task, sentence_avg): - super().__init__(task) - self.sentence_avg = sentence_avg - - @classmethod - def build_criterion(cls, cfg: AdaptiveLossConfig, task): - if cfg.ddp_backend in {"c10d", "pytorch_ddp"}: - raise Exception( - "AdaptiveLoss is not compatible with the PyTorch " - "version of DistributedDataParallel. Please use " - "`--ddp-backend=legacy_ddp` instead." - ) - return cls(task, cfg.sentence_avg) - - def forward(self, model, sample, reduce=True): - """Compute the loss for the given sample. - - Returns a tuple with three elements: - 1) the loss - 2) the sample size, which is used as the denominator for the gradient - 3) logging outputs to display while training - """ - - assert ( - hasattr(model.decoder, "adaptive_softmax") - and model.decoder.adaptive_softmax is not None - ) - adaptive_softmax = model.decoder.adaptive_softmax - - net_output = model(**sample["net_input"]) - orig_target = model.get_targets(sample, net_output) - - nsentences = orig_target.size(0) - orig_target = orig_target.view(-1) - - bsz = orig_target.size(0) - - logits, target = adaptive_softmax(net_output[0], orig_target) - assert len(target) == len(logits) - - loss = net_output[0].new(1 if reduce else bsz).zero_() - - for i in range(len(target)): - if target[i] is not None: - assert target[i].min() >= 0 and target[i].max() <= logits[i].size(1) - loss += F.cross_entropy( - logits[i], - target[i], - ignore_index=self.padding_idx, - reduction="sum" if reduce else "none", - ) - - orig = utils.strip_pad(orig_target, self.padding_idx) - ntokens = orig.numel() - sample_size = sample["target"].size(0) if self.sentence_avg else ntokens - logging_output = { - "loss": loss.data, - "ntokens": ntokens, - "nsentences": nsentences, - "sample_size": sample_size, - } - return loss, sample_size, logging_output - - @staticmethod - def reduce_metrics(logging_outputs) -> None: - """Aggregate logging outputs from data parallel training.""" - loss_sum = utils.item(sum(log.get("loss", 0) for log in logging_outputs)) - ntokens = utils.item(sum(log.get("ntokens", 0) for log in logging_outputs)) - sample_size = utils.item( - sum(log.get("sample_size", 0) for log in logging_outputs) - ) - - metrics.log_scalar( - "loss", loss_sum / sample_size / math.log(2), sample_size, round=3 - ) - if sample_size != ntokens: - metrics.log_scalar( - "nll_loss", loss_sum / ntokens / math.log(2), ntokens, round=3 - ) - metrics.log_derived( - "ppl", lambda meters: utils.get_perplexity(meters["nll_loss"].avg) - ) - else: - metrics.log_derived( - "ppl", lambda meters: utils.get_perplexity(meters["loss"].avg) - ) - - @staticmethod - def logging_outputs_can_be_summed() -> bool: - """ - Whether the logging outputs returned by `forward` can be summed - across workers prior to calling `reduce_metrics`. Setting this - to True will improves distributed training speed. - """ - return True diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/composite_loss.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/composite_loss.py deleted file mode 100644 index 98e835fa6e4c0bcad062df9c519701bf795c98be..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/composite_loss.py +++ /dev/null @@ -1,100 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from fairseq import utils -from fairseq.criterions import LegacyFairseqCriterion, register_criterion -from torch import nn - - -@register_criterion("composite_loss") -class CompositeLoss(LegacyFairseqCriterion): - """This is a composite loss that, given a list of model outputs and a list of targets, - computes an average of losses for each output-target pair""" - - def __init__(self, args, task): - super().__init__(args, task) - self.underlying_criterion = args.underlying_criterion - - @staticmethod - def add_args(parser): - """Add criterion-specific arguments to the parser.""" - # fmt: off - parser.add_argument('--underlying-criterion', type=str, metavar='VAL', required=True, - help='underlying criterion to use for the composite loss') - # fmt: on - - @staticmethod - def build_underlying_criterion(args, task): - saved_criterion = args.criterion - args.criterion = args.underlying_criterion - assert saved_criterion != args.underlying_criterion - underlying_criterion = task.build_criterion(args) - args.criterion = saved_criterion - return underlying_criterion - - @classmethod - def build_criterion(cls, args, task): - underlying_criterion = CompositeLoss.build_underlying_criterion(args, task) - - class FakeModel(nn.Module): - def __init__(self, model, net_out, target): - super().__init__() - self.model = model - self.net_out = net_out - self.target = target - - def forward(self, **unused): - return self.net_out - - def get_normalized_probs(self, net_output, log_probs, sample=None): - return self.model.get_normalized_probs( - net_output, log_probs, sample=sample - ) - - def get_targets(self, *unused): - return self.target - - @property - def decoder(self): - return self.model.decoder - - class _CompositeLoss(LegacyFairseqCriterion): - def __init__(self, args, task, underlying_criterion): - super().__init__(args, task) - self.underlying_criterion = underlying_criterion - - def forward(self, model, sample, reduce=True): - net_outputs = model(**sample["net_input"]) - targets = sample["target"] - - bsz = targets[0].size(0) - loss = net_outputs[0][0].new(1 if reduce else bsz).float().zero_() - - sample_size = 0 - logging_output = {} - for o, t in zip(net_outputs[0], targets): - m = FakeModel(model, (o, net_outputs[1]), t) - sample["target"] = t - l, ss, logging_output = self.underlying_criterion(m, sample, reduce) - loss += l - sample_size += ss - - loss.div_(len(targets)) - sample_size /= len(targets) - - logging_output["loss"] = utils.item(loss.data) if reduce else loss.data - return loss, sample_size, logging_output - - @staticmethod - def aggregate_logging_outputs(logging_outputs): - return underlying_criterion.__class__.aggregate_logging_outputs( - logging_outputs - ) - - @staticmethod - def reduce_metrics(logging_outputs) -> None: - underlying_criterion.__class__.reduce_metrics(logging_outputs) - - return _CompositeLoss(args, task, underlying_criterion) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/cross_entropy.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/cross_entropy.py deleted file mode 100644 index fe461064716b38ecf2eb610daddbb609a1884e6b..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/cross_entropy.py +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import math -from dataclasses import dataclass - -import torch.nn.functional as F -from fairseq import metrics, utils -from fairseq.criterions import FairseqCriterion, register_criterion -from fairseq.dataclass import FairseqDataclass -from omegaconf import II - - -@dataclass -class CrossEntropyCriterionConfig(FairseqDataclass): - sentence_avg: bool = II("optimization.sentence_avg") - - -@register_criterion("cross_entropy", dataclass=CrossEntropyCriterionConfig) -class CrossEntropyCriterion(FairseqCriterion): - def __init__(self, task, sentence_avg): - super().__init__(task) - self.sentence_avg = sentence_avg - - def forward(self, model, sample, reduce=True): - """Compute the loss for the given sample. - - Returns a tuple with three elements: - 1) the loss - 2) the sample size, which is used as the denominator for the gradient - 3) logging outputs to display while training - """ - net_output = model(**sample["net_input"]) - loss, _ = self.compute_loss(model, net_output, sample, reduce=reduce) - sample_size = ( - sample["target"].size(0) if self.sentence_avg else sample["ntokens"] - ) - logging_output = { - "loss": loss.data, - "ntokens": sample["ntokens"], - "nsentences": sample["target"].size(0), - "sample_size": sample_size, - } - return loss, sample_size, logging_output - - def compute_loss(self, model, net_output, sample, reduce=True): - lprobs = model.get_normalized_probs(net_output, log_probs=True) - lprobs = lprobs.view(-1, lprobs.size(-1)) - target = model.get_targets(sample, net_output).view(-1) - loss = F.nll_loss( - lprobs, - target, - ignore_index=self.padding_idx, - reduction="sum" if reduce else "none", - ) - return loss, loss - - @staticmethod - def reduce_metrics(logging_outputs) -> None: - """Aggregate logging outputs from data parallel training.""" - loss_sum = sum(log.get("loss", 0) for log in logging_outputs) - ntokens = sum(log.get("ntokens", 0) for log in logging_outputs) - sample_size = sum(log.get("sample_size", 0) for log in logging_outputs) - - # we divide by log(2) to convert the loss from base e to base 2 - metrics.log_scalar( - "loss", loss_sum / sample_size / math.log(2), sample_size, round=3 - ) - if sample_size != ntokens: - metrics.log_scalar( - "nll_loss", loss_sum / ntokens / math.log(2), ntokens, round=3 - ) - metrics.log_derived( - "ppl", lambda meters: utils.get_perplexity(meters["nll_loss"].avg) - ) - else: - metrics.log_derived( - "ppl", lambda meters: utils.get_perplexity(meters["loss"].avg) - ) - - @staticmethod - def logging_outputs_can_be_summed() -> bool: - """ - Whether the logging outputs returned by `forward` can be summed - across workers prior to calling `reduce_metrics`. Setting this - to True will improves distributed training speed. - """ - return True diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/ctc.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/ctc.py deleted file mode 100644 index 73a8165c6cb765f82a1952cbece3a782a6e8f0c3..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/ctc.py +++ /dev/null @@ -1,295 +0,0 @@ -# All rights reserved. -# -# This source code is licensed under the license found in the LICENSE file in -# the root directory of this source tree. An additional grant of patent rights -# can be found in the PATENTS file in the same directory. - -import math -from argparse import Namespace -from dataclasses import dataclass, field -from typing import Optional - -import torch -import torch.nn.functional as F -from fairseq import metrics, utils -from fairseq.criterions import FairseqCriterion, register_criterion -from fairseq.data.data_utils import post_process -from fairseq.dataclass import FairseqDataclass -from fairseq.logging.meters import safe_round -from fairseq.tasks import FairseqTask -from omegaconf import II - - -@dataclass -class CtcCriterionConfig(FairseqDataclass): - zero_infinity: bool = field( - default=False, - metadata={"help": "zero inf loss when source length <= target length"}, - ) - sentence_avg: bool = II("optimization.sentence_avg") - post_process: str = field( - default="letter", - metadata={ - "help": "how to post process predictions into words. can be letter, " - "wordpiece, BPE symbols, etc. " - "See fairseq.data.data_utils.post_process() for full list of options" - }, - ) - wer_kenlm_model: Optional[str] = field( - default=None, - metadata={ - "help": "if this is provided, use kenlm to compute wer (along with other wer_* args)" - }, - ) - wer_lexicon: Optional[str] = field( - default=None, - metadata={"help": "lexicon to use with wer_kenlm_model"}, - ) - wer_lm_weight: float = field( - default=2.0, - metadata={"help": "lm weight to use with wer_kenlm_model"}, - ) - wer_word_score: float = field( - default=-1.0, - metadata={"help": "lm word score to use with wer_kenlm_model"}, - ) - - wer_args: Optional[str] = field( - default=None, - metadata={ - "help": "DEPRECATED: tuple of (wer_kenlm_model, wer_lexicon, wer_lm_weight, wer_word_score)" - }, - ) - - -@register_criterion("ctc", dataclass=CtcCriterionConfig) -class CtcCriterion(FairseqCriterion): - def __init__(self, cfg: CtcCriterionConfig, task: FairseqTask): - super().__init__(task) - self.blank_idx = ( - task.target_dictionary.index(task.blank_symbol) - if hasattr(task, "blank_symbol") - else 0 - ) - self.pad_idx = task.target_dictionary.pad() - self.eos_idx = task.target_dictionary.eos() - self.post_process = cfg.post_process - - if cfg.wer_args is not None: - ( - cfg.wer_kenlm_model, - cfg.wer_lexicon, - cfg.wer_lm_weight, - cfg.wer_word_score, - ) = eval(cfg.wer_args) - - if cfg.wer_kenlm_model is not None and cfg.wer_kenlm_model != "": - from examples.speech_recognition.w2l_decoder import W2lKenLMDecoder - - dec_args = Namespace() - dec_args.nbest = 1 - dec_args.criterion = "ctc" - dec_args.kenlm_model = cfg.wer_kenlm_model - dec_args.lexicon = cfg.wer_lexicon - dec_args.beam = 50 - dec_args.beam_size_token = min(50, len(task.target_dictionary)) - dec_args.beam_threshold = min(50, len(task.target_dictionary)) - dec_args.lm_weight = cfg.wer_lm_weight - dec_args.word_score = cfg.wer_word_score - dec_args.unk_weight = -math.inf - dec_args.sil_weight = 0 - - self.w2l_decoder = W2lKenLMDecoder(dec_args, task.target_dictionary) - else: - self.w2l_decoder = None - - self.zero_infinity = cfg.zero_infinity - self.sentence_avg = cfg.sentence_avg - - def forward(self, model, sample, reduce=True): - net_output = model(**sample["net_input"]) - lprobs = model.get_normalized_probs( - net_output, log_probs=True - ).contiguous() # (T, B, C) from the encoder - - if "src_lengths" in sample["net_input"]: - input_lengths = sample["net_input"]["src_lengths"] - else: - if net_output["padding_mask"] is not None: - non_padding_mask = ~net_output["padding_mask"] - input_lengths = non_padding_mask.long().sum(-1) - else: - input_lengths = lprobs.new_full( - (lprobs.size(1),), lprobs.size(0), dtype=torch.long - ) - - pad_mask = (sample["target"] != self.pad_idx) & ( - sample["target"] != self.eos_idx - ) - targets_flat = sample["target"].masked_select(pad_mask) - if "target_lengths" in sample: - target_lengths = sample["target_lengths"] - else: - target_lengths = pad_mask.sum(-1) - - with torch.backends.cudnn.flags(enabled=False): - loss = F.ctc_loss( - lprobs, - targets_flat, - input_lengths, - target_lengths, - blank=self.blank_idx, - reduction="sum", - zero_infinity=self.zero_infinity, - ) - - ntokens = ( - sample["ntokens"] if "ntokens" in sample else target_lengths.sum().item() - ) - - sample_size = sample["target"].size(0) if self.sentence_avg else ntokens - logging_output = { - "loss": utils.item(loss.data), # * sample['ntokens'], - "ntokens": ntokens, - "nsentences": sample["id"].numel(), - "sample_size": sample_size, - } - - if not model.training: - import editdistance - - with torch.no_grad(): - lprobs_t = lprobs.transpose(0, 1).float().contiguous().cpu() - - c_err = 0 - c_len = 0 - w_errs = 0 - w_len = 0 - wv_errs = 0 - for lp, t, inp_l in zip( - lprobs_t, - sample["target_label"] - if "target_label" in sample - else sample["target"], - input_lengths, - ): - lp = lp[:inp_l].unsqueeze(0) - - decoded = None - if self.w2l_decoder is not None: - decoded = self.w2l_decoder.decode(lp) - if len(decoded) < 1: - decoded = None - else: - decoded = decoded[0] - if len(decoded) < 1: - decoded = None - else: - decoded = decoded[0] - - p = (t != self.task.target_dictionary.pad()) & ( - t != self.task.target_dictionary.eos() - ) - targ = t[p] - targ_units = self.task.target_dictionary.string(targ) - targ_units_arr = targ.tolist() - - toks = lp.argmax(dim=-1).unique_consecutive() - pred_units_arr = toks[toks != self.blank_idx].tolist() - - c_err += editdistance.eval(pred_units_arr, targ_units_arr) - c_len += len(targ_units_arr) - - targ_words = post_process(targ_units, self.post_process).split() - - pred_units = self.task.target_dictionary.string(pred_units_arr) - pred_words_raw = post_process(pred_units, self.post_process).split() - - if decoded is not None and "words" in decoded: - pred_words = decoded["words"] - w_errs += editdistance.eval(pred_words, targ_words) - wv_errs += editdistance.eval(pred_words_raw, targ_words) - else: - dist = editdistance.eval(pred_words_raw, targ_words) - w_errs += dist - wv_errs += dist - - w_len += len(targ_words) - - logging_output["wv_errors"] = wv_errs - logging_output["w_errors"] = w_errs - logging_output["w_total"] = w_len - logging_output["c_errors"] = c_err - logging_output["c_total"] = c_len - - return loss, sample_size, logging_output - - @staticmethod - def reduce_metrics(logging_outputs) -> None: - """Aggregate logging outputs from data parallel training.""" - - loss_sum = utils.item(sum(log.get("loss", 0) for log in logging_outputs)) - ntokens = utils.item(sum(log.get("ntokens", 0) for log in logging_outputs)) - nsentences = utils.item( - sum(log.get("nsentences", 0) for log in logging_outputs) - ) - sample_size = utils.item( - sum(log.get("sample_size", 0) for log in logging_outputs) - ) - - metrics.log_scalar( - "loss", loss_sum / sample_size / math.log(2), sample_size, round=3 - ) - metrics.log_scalar("ntokens", ntokens) - metrics.log_scalar("nsentences", nsentences) - if sample_size != ntokens: - metrics.log_scalar( - "nll_loss", loss_sum / ntokens / math.log(2), ntokens, round=3 - ) - - c_errors = sum(log.get("c_errors", 0) for log in logging_outputs) - metrics.log_scalar("_c_errors", c_errors) - c_total = sum(log.get("c_total", 0) for log in logging_outputs) - metrics.log_scalar("_c_total", c_total) - w_errors = sum(log.get("w_errors", 0) for log in logging_outputs) - metrics.log_scalar("_w_errors", w_errors) - wv_errors = sum(log.get("wv_errors", 0) for log in logging_outputs) - metrics.log_scalar("_wv_errors", wv_errors) - w_total = sum(log.get("w_total", 0) for log in logging_outputs) - metrics.log_scalar("_w_total", w_total) - - if c_total > 0: - metrics.log_derived( - "uer", - lambda meters: safe_round( - meters["_c_errors"].sum * 100.0 / meters["_c_total"].sum, 3 - ) - if meters["_c_total"].sum > 0 - else float("nan"), - ) - if w_total > 0: - metrics.log_derived( - "wer", - lambda meters: safe_round( - meters["_w_errors"].sum * 100.0 / meters["_w_total"].sum, 3 - ) - if meters["_w_total"].sum > 0 - else float("nan"), - ) - metrics.log_derived( - "raw_wer", - lambda meters: safe_round( - meters["_wv_errors"].sum * 100.0 / meters["_w_total"].sum, 3 - ) - if meters["_w_total"].sum > 0 - else float("nan"), - ) - - @staticmethod - def logging_outputs_can_be_summed() -> bool: - """ - Whether the logging outputs returned by `forward` can be summed - across workers prior to calling `reduce_metrics`. Setting this - to True will improves distributed training speed. - """ - return True diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/fairseq_criterion.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/fairseq_criterion.py deleted file mode 100644 index ff4beb02503ea48a6c09596630aad4c710be94b6..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/fairseq_criterion.py +++ /dev/null @@ -1,120 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import inspect -from typing import Any, Dict, List - -from fairseq import metrics, utils -from fairseq.dataclass import FairseqDataclass -from fairseq.dataclass.utils import gen_parser_from_dataclass -from torch.nn.modules.loss import _Loss - - -class FairseqCriterion(_Loss): - def __init__(self, task): - super().__init__() - self.task = task - if hasattr(task, "target_dictionary"): - tgt_dict = task.target_dictionary - self.padding_idx = tgt_dict.pad() if tgt_dict is not None else -100 - - @classmethod - def add_args(cls, parser): - """Add criterion-specific arguments to the parser.""" - dc = getattr(cls, "__dataclass", None) - if dc is not None: - gen_parser_from_dataclass(parser, dc()) - - @classmethod - def build_criterion(cls, cfg: FairseqDataclass, task): - """Construct a criterion from command-line args.""" - # arguments in the __init__. - init_args = {} - for p in inspect.signature(cls).parameters.values(): - if ( - p.kind == p.POSITIONAL_ONLY - or p.kind == p.VAR_POSITIONAL - or p.kind == p.VAR_KEYWORD - ): - # we haven't implemented inference for these argument types, - # but PRs welcome :) - raise NotImplementedError("{} not supported".format(p.kind)) - - assert p.kind in {p.POSITIONAL_OR_KEYWORD, p.KEYWORD_ONLY} - - if p.name == "task": - init_args["task"] = task - elif p.name == "cfg": - init_args["cfg"] = cfg - elif hasattr(cfg, p.name): - init_args[p.name] = getattr(cfg, p.name) - elif p.default != p.empty: - pass # we'll use the default value - else: - raise NotImplementedError( - "Unable to infer Criterion arguments, please implement " - "{}.build_criterion".format(cls.__name__) - ) - return cls(**init_args) - - def forward(self, model, sample, reduce=True): - """Compute the loss for the given sample. - - Returns a tuple with three elements: - 1) the loss - 2) the sample size, which is used as the denominator for the gradient - 3) logging outputs to display while training - """ - raise NotImplementedError - - @staticmethod - def aggregate_logging_outputs( - logging_outputs: List[Dict[str, Any]] - ) -> Dict[str, Any]: - """Aggregate logging outputs from data parallel training.""" - utils.deprecation_warning( - "The aggregate_logging_outputs API is deprecated. " - "Please use the reduce_metrics API instead." - ) - raise NotImplementedError - - @classmethod - def reduce_metrics(cls, logging_outputs: List[Dict[str, Any]]) -> None: - """Aggregate logging outputs from data parallel training.""" - utils.deprecation_warning( - "Criterions should implement the reduce_metrics API. " - "Falling back to deprecated aggregate_logging_outputs API." - ) - agg_logging_outputs = cls.aggregate_logging_outputs(logging_outputs) - for k, v in agg_logging_outputs.items(): - if k in {"nsentences", "ntokens", "sample_size"}: - continue - metrics.log_scalar(k, v) - - @staticmethod - def logging_outputs_can_be_summed() -> bool: - """ - Whether the logging outputs returned by `forward` can be summed - across workers prior to calling `reduce_metrics`. Setting this - to True will improves distributed training speed. - """ - return False - - -class LegacyFairseqCriterion(FairseqCriterion): - def __init__(self, args, task): - super().__init__(task=task) - self.args = args - - utils.deprecation_warning( - "Criterions should take explicit arguments instead of an " - "argparse.Namespace object, please update your criterion by " - "extending FairseqCriterion instead of LegacyFairseqCriterion." - ) - - @classmethod - def build_criterion(cls, args, task): - """Construct a criterion from command-line args.""" - return cls(args, task) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/fastspeech2_loss.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/fastspeech2_loss.py deleted file mode 100644 index db213a8da9c1606a6e1a72f2bb3995def331a6e9..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/fastspeech2_loss.py +++ /dev/null @@ -1,135 +0,0 @@ -# Copyright (c) 2017-present, Facebook, Inc. -# All rights reserved. -# -# This source code is licensed under the license found in the LICENSE file in -# the root directory of this source tree. An additional grant of patent rights -# can be found in the PATENTS file in the same directory. - -from dataclasses import dataclass, field -from typing import Any, Dict, List - -import torch -import torch.nn.functional as F -from fairseq import metrics, utils -from fairseq.criterions import FairseqCriterion, register_criterion -from fairseq.data.data_utils import lengths_to_mask -from fairseq.dataclass import FairseqDataclass -from fairseq.models.fairseq_model import FairseqEncoderModel - - -@dataclass -class FastSpeech2CriterionConfig(FairseqDataclass): - ctc_weight: float = field(default=0.0, metadata={"help": "weight for CTC loss"}) - - -@register_criterion("fastspeech2", dataclass=FastSpeech2CriterionConfig) -class FastSpeech2Loss(FairseqCriterion): - def __init__(self, task, ctc_weight): - super().__init__(task) - self.ctc_weight = ctc_weight - - def forward(self, model: FairseqEncoderModel, sample, reduction="mean"): - src_tokens = sample["net_input"]["src_tokens"] - src_lens = sample["net_input"]["src_lengths"] - tgt_lens = sample["target_lengths"] - _feat_out, _feat_out_post, _, log_dur_out, pitch_out, energy_out = model( - src_tokens=src_tokens, - src_lengths=src_lens, - prev_output_tokens=sample["net_input"]["prev_output_tokens"], - incremental_state=None, - target_lengths=tgt_lens, - speaker=sample["speaker"], - durations=sample["durations"], - pitches=sample["pitches"], - energies=sample["energies"], - ) - - src_mask = lengths_to_mask(sample["net_input"]["src_lengths"]) - tgt_mask = lengths_to_mask(sample["target_lengths"]) - - pitches, energies = sample["pitches"], sample["energies"] - pitch_out, pitches = pitch_out[src_mask], pitches[src_mask] - energy_out, energies = energy_out[src_mask], energies[src_mask] - - feat_out, feat = _feat_out[tgt_mask], sample["target"][tgt_mask] - l1_loss = F.l1_loss(feat_out, feat, reduction=reduction) - if _feat_out_post is not None: - l1_loss += F.l1_loss(_feat_out_post[tgt_mask], feat, reduction=reduction) - - pitch_loss = F.mse_loss(pitch_out, pitches, reduction=reduction) - energy_loss = F.mse_loss(energy_out, energies, reduction=reduction) - - log_dur_out = log_dur_out[src_mask] - dur = sample["durations"].float() - dur = dur.half() if log_dur_out.type().endswith(".HalfTensor") else dur - log_dur = torch.log(dur + 1)[src_mask] - dur_loss = F.mse_loss(log_dur_out, log_dur, reduction=reduction) - - ctc_loss = torch.tensor(0.0).type_as(l1_loss) - if self.ctc_weight > 0.0: - lprobs = model.get_normalized_probs((_feat_out,), log_probs=True) - lprobs = lprobs.transpose(0, 1) # T x B x C - src_mask = lengths_to_mask(src_lens) - src_tokens_flat = src_tokens.masked_select(src_mask) - ctc_loss = ( - F.ctc_loss( - lprobs, - src_tokens_flat, - tgt_lens, - src_lens, - reduction=reduction, - zero_infinity=True, - ) - * self.ctc_weight - ) - - loss = l1_loss + dur_loss + pitch_loss + energy_loss + ctc_loss - - sample_size = sample["nsentences"] - logging_output = { - "loss": utils.item(loss.data), - "ntokens": sample["ntokens"], - "nsentences": sample["nsentences"], - "sample_size": sample_size, - "l1_loss": utils.item(l1_loss.data), - "dur_loss": utils.item(dur_loss.data), - "pitch_loss": utils.item(pitch_loss.data), - "energy_loss": utils.item(energy_loss.data), - "ctc_loss": utils.item(ctc_loss.data), - } - return loss, sample_size, logging_output - - @classmethod - def reduce_metrics(cls, logging_outputs: List[Dict[str, Any]]) -> None: - ns = [log.get("sample_size", 0) for log in logging_outputs] - ntot = sum(ns) - ws = [n / (ntot + 1e-8) for n in ns] - for key in [ - "loss", - "l1_loss", - "dur_loss", - "pitch_loss", - "energy_loss", - "ctc_loss", - ]: - vals = [log.get(key, 0) for log in logging_outputs] - val = sum(val * w for val, w in zip(vals, ws)) - metrics.log_scalar(key, val, ntot, round=3) - metrics.log_scalar("sample_size", ntot, len(logging_outputs)) - - # inference metrics - if "targ_frames" not in logging_outputs[0]: - return - n = sum(log.get("targ_frames", 0) for log in logging_outputs) - for key, new_key in [ - ("mcd_loss", "mcd_loss"), - ("pred_frames", "pred_ratio"), - ("nins", "ins_rate"), - ("ndel", "del_rate"), - ]: - val = sum(log.get(key, 0) for log in logging_outputs) - metrics.log_scalar(new_key, val / n, n, round=3) - - @staticmethod - def logging_outputs_can_be_summed() -> bool: - return False diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/hubert_criterion.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/hubert_criterion.py deleted file mode 100644 index 83b514aeefb4c074076af80994194b236c49d283..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/hubert_criterion.py +++ /dev/null @@ -1,194 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import math -import re -from dataclasses import dataclass, field -from typing import List, Optional - -import torch -import torch.nn.functional as F -from fairseq import metrics, utils -from fairseq.criterions import FairseqCriterion, register_criterion -from fairseq.dataclass import FairseqDataclass - - -@dataclass -class HubertCriterionConfig(FairseqDataclass): - pred_masked_weight: float = field( - default=1.0, - metadata={"help": "weight for predictive loss for masked frames"}, - ) - pred_nomask_weight: float = field( - default=0.0, - metadata={"help": "weight for predictive loss for unmasked frames"}, - ) - loss_weights: Optional[List[float]] = field( - default=None, - metadata={"help": "weights for additional loss terms (not first one)"}, - ) - log_keys: List[str] = field( - default_factory=lambda: [], - metadata={"help": "output keys to log"}, - ) - - -@register_criterion("hubert", dataclass=HubertCriterionConfig) -class HubertCriterion(FairseqCriterion): - def __init__( - self, - task, - pred_masked_weight, - pred_nomask_weight, - loss_weights=None, - log_keys=None, - ): - super().__init__(task) - self.pred_masked_weight = pred_masked_weight - self.pred_nomask_weight = pred_nomask_weight - self.loss_weights = loss_weights - self.log_keys = [] if log_keys is None else log_keys - - def forward(self, model, sample, reduce=True, log_pred=False): - """Compute the loss for the given sample. - Returns a tuple with three elements: - 1) the loss - 2) the sample size, which is used as the denominator for the gradient - 3) logging outputs to display while training - """ - net_output = model(target_list=sample["target_list"], **sample["net_input"]) - loss = 0.0 - sample_size = 0 - logging_output = {} - reduction = "sum" if reduce else "none" - - loss_m_list = [] - logp_m_list = model.get_logits(net_output, True) - targ_m_list = model.get_targets(net_output, True) - assert self.pred_masked_weight == 0 or len(logp_m_list) > 0 - for i, (logp_m, targ_m) in enumerate(zip(logp_m_list, targ_m_list)): - loss_m = F.cross_entropy(logp_m, targ_m, reduction=reduction) - loss_m_list.append(loss_m) - logging_output[f"loss_m_{i}"] = loss_m.detach().item() - if self.pred_masked_weight > 0: - loss += self.pred_masked_weight * sum(loss_m_list) - sample_size += targ_m_list[0].numel() - - loss_u_list = [] - logp_u_list = model.get_logits(net_output, False) - targ_u_list = model.get_targets(net_output, False) - assert self.pred_nomask_weight == 0 or len(logp_u_list) > 0 - for i, (logp_u, targ_u) in enumerate(zip(logp_u_list, targ_u_list)): - loss_u = F.cross_entropy(logp_u, targ_u, reduction=reduction) - loss_u_list.append(loss_u) - logging_output[f"loss_u_{i}"] = loss_u.detach().item() - if self.pred_nomask_weight > 0: - loss += self.pred_nomask_weight * sum(loss_u_list) - sample_size += targ_u_list[0].numel() - - if self.loss_weights is not None: - assert hasattr(model, "get_extra_losses") - extra_losses, names = model.get_extra_losses(net_output) - if torch.is_tensor(extra_losses): - extra_losses = [extra_losses] - names = [names] - if len(self.loss_weights) == 1 and len(extra_losses) != 1: - self.loss_weights = [self.loss_weights[0]] * len(extra_losses) - assert len(extra_losses) == len( - self.loss_weights - ), f"{len(extra_losses)}, {len(self.loss_weights)}" - for p, n, coef in zip(extra_losses, names, self.loss_weights): - if coef != 0 and p is not None: - p = coef * p.float() * sample_size - loss += p - logging_output[f"loss_{n}"] = p.item() - - logging_output = { - "loss": loss.item() if reduce else loss, - "ntokens": sample_size, - "nsentences": sample["id"].numel(), - "sample_size": sample_size, - **logging_output, - } - - for lk in self.log_keys: - if lk in net_output: - logging_output[lk] = float((net_output[lk])) - - def compute_correct(logits): - if logits.numel() == 0: - return 0, 0 - else: - assert logits.dim() > 1, logits.shape - max = logits.argmax(-1) == 0 - min = logits.argmin(-1) == 0 - both = max & min - corr = max.long().sum().item() - both.long().sum().item() - count = max.numel() - return corr, count - - with torch.no_grad(): - for i, logp_m in enumerate(logp_m_list): - corr_m, count_m = compute_correct(logp_m) - logging_output[f"correct_m_{i}"] = corr_m - logging_output[f"count_m_{i}"] = count_m - - for i, logp_u in enumerate(logp_u_list): - corr_u, count_u = compute_correct(logp_u) - logging_output[f"correct_u_{i}"] = corr_u - logging_output[f"count_u_{i}"] = count_u - - return loss, sample_size, logging_output - - @staticmethod - def reduce_metrics(logging_outputs) -> None: - """Aggregate logging outputs from data parallel training (copied from normal cross entropy).""" - loss_sum = sum(log.get("loss", 0) for log in logging_outputs) - ntokens = sum(log.get("ntokens", 0) for log in logging_outputs) - sample_size = sum(log.get("sample_size", 0) for log in logging_outputs) - - metrics.log_scalar( - "loss", loss_sum / sample_size / math.log(2), sample_size, round=3 - ) - if sample_size != ntokens: - metrics.log_scalar( - "nll_loss", loss_sum / ntokens / math.log(2), ntokens, round=3 - ) - metrics.log_derived( - "ppl", lambda meters: utils.get_perplexity(meters["nll_loss"].avg) - ) - else: - metrics.log_derived( - "ppl", lambda meters: utils.get_perplexity(meters["loss"].avg) - ) - - counts = {} - for lk in logging_outputs[0].keys(): - if lk.startswith("count_"): - val = sum(log[lk] for log in logging_outputs) - metrics.log_scalar(lk, val) - counts[lk] = val - - for lk in logging_outputs[0].keys(): - if lk.startswith("loss_"): - val = sum(log[lk] for log in logging_outputs) - metrics.log_scalar(lk, val / sample_size / math.log(2), round=3) - elif lk.startswith("correct_"): - val = sum(log[lk] for log in logging_outputs) - metrics.log_scalar(lk, val / counts[re.sub("correct", "count", lk)]) - - @staticmethod - def aggregate_logging_outputs(logging_outputs): - """Aggregate logging outputs from data parallel training.""" - raise NotImplementedError() - - @staticmethod - def logging_outputs_can_be_summed() -> bool: - """ - Whether the logging outputs returned by `forward` can be summed - across workers prior to calling `reduce_metrics`. Setting this - to True will improves distributed training speed. - """ - return False diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/label_smoothed_cross_entropy.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/label_smoothed_cross_entropy.py deleted file mode 100644 index cb43be0ca549ab2c120ef520617931fec6b17c41..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/label_smoothed_cross_entropy.py +++ /dev/null @@ -1,167 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import math -from dataclasses import dataclass, field - -import torch -from fairseq import metrics, utils -from fairseq.criterions import FairseqCriterion, register_criterion -from fairseq.dataclass import FairseqDataclass -from omegaconf import II - - -@dataclass -class LabelSmoothedCrossEntropyCriterionConfig(FairseqDataclass): - label_smoothing: float = field( - default=0.0, - metadata={"help": "epsilon for label smoothing, 0 means no label smoothing"}, - ) - report_accuracy: bool = field( - default=False, - metadata={"help": "report accuracy metric"}, - ) - ignore_prefix_size: int = field( - default=0, - metadata={"help": "Ignore first N tokens"}, - ) - sentence_avg: bool = II("optimization.sentence_avg") - - -def label_smoothed_nll_loss(lprobs, target, epsilon, ignore_index=None, reduce=True): - if target.dim() == lprobs.dim() - 1: - target = target.unsqueeze(-1) - nll_loss = -lprobs.gather(dim=-1, index=target) - smooth_loss = -lprobs.sum(dim=-1, keepdim=True) - if ignore_index is not None: - pad_mask = target.eq(ignore_index) - nll_loss.masked_fill_(pad_mask, 0.0) - smooth_loss.masked_fill_(pad_mask, 0.0) - else: - nll_loss = nll_loss.squeeze(-1) - smooth_loss = smooth_loss.squeeze(-1) - if reduce: - nll_loss = nll_loss.sum() - smooth_loss = smooth_loss.sum() - eps_i = epsilon / (lprobs.size(-1) - 1) - loss = (1.0 - epsilon - eps_i) * nll_loss + eps_i * smooth_loss - return loss, nll_loss - - -@register_criterion( - "label_smoothed_cross_entropy", dataclass=LabelSmoothedCrossEntropyCriterionConfig -) -class LabelSmoothedCrossEntropyCriterion(FairseqCriterion): - def __init__( - self, - task, - sentence_avg, - label_smoothing, - ignore_prefix_size=0, - report_accuracy=False, - ): - super().__init__(task) - self.sentence_avg = sentence_avg - self.eps = label_smoothing - self.ignore_prefix_size = ignore_prefix_size - self.report_accuracy = report_accuracy - - def forward(self, model, sample, reduce=True): - """Compute the loss for the given sample. - - Returns a tuple with three elements: - 1) the loss - 2) the sample size, which is used as the denominator for the gradient - 3) logging outputs to display while training - """ - net_output = model(**sample["net_input"]) - loss, nll_loss = self.compute_loss(model, net_output, sample, reduce=reduce) - sample_size = ( - sample["target"].size(0) if self.sentence_avg else sample["ntokens"] - ) - logging_output = { - "loss": loss.data, - "nll_loss": nll_loss.data, - "ntokens": sample["ntokens"], - "nsentences": sample["target"].size(0), - "sample_size": sample_size, - } - if self.report_accuracy: - n_correct, total = self.compute_accuracy(model, net_output, sample) - logging_output["n_correct"] = utils.item(n_correct.data) - logging_output["total"] = utils.item(total.data) - return loss, sample_size, logging_output - - def get_lprobs_and_target(self, model, net_output, sample): - lprobs = model.get_normalized_probs(net_output, log_probs=True) - target = model.get_targets(sample, net_output) - if self.ignore_prefix_size > 0: - # lprobs: B x T x C - lprobs = lprobs[:, self.ignore_prefix_size :, :].contiguous() - target = target[:, self.ignore_prefix_size :].contiguous() - return lprobs.view(-1, lprobs.size(-1)), target.view(-1) - - def compute_loss(self, model, net_output, sample, reduce=True): - lprobs, target = self.get_lprobs_and_target(model, net_output, sample) - loss, nll_loss = label_smoothed_nll_loss( - lprobs, - target, - self.eps, - ignore_index=self.padding_idx, - reduce=reduce, - ) - return loss, nll_loss - - def compute_accuracy(self, model, net_output, sample): - lprobs, target = self.get_lprobs_and_target(model, net_output, sample) - mask = target.ne(self.padding_idx) - n_correct = torch.sum( - lprobs.argmax(1).masked_select(mask).eq(target.masked_select(mask)) - ) - total = torch.sum(mask) - return n_correct, total - - @classmethod - def reduce_metrics(cls, logging_outputs) -> None: - """Aggregate logging outputs from data parallel training.""" - loss_sum = sum(log.get("loss", 0) for log in logging_outputs) - nll_loss_sum = sum(log.get("nll_loss", 0) for log in logging_outputs) - ntokens = sum(log.get("ntokens", 0) for log in logging_outputs) - sample_size = sum(log.get("sample_size", 0) for log in logging_outputs) - - metrics.log_scalar( - "loss", loss_sum / sample_size / math.log(2), sample_size, round=3 - ) - metrics.log_scalar( - "nll_loss", nll_loss_sum / ntokens / math.log(2), ntokens, round=3 - ) - metrics.log_derived( - "ppl", lambda meters: utils.get_perplexity(meters["nll_loss"].avg) - ) - - total = utils.item(sum(log.get("total", 0) for log in logging_outputs)) - if total > 0: - metrics.log_scalar("total", total) - n_correct = utils.item( - sum(log.get("n_correct", 0) for log in logging_outputs) - ) - metrics.log_scalar("n_correct", n_correct) - metrics.log_derived( - "accuracy", - lambda meters: round( - meters["n_correct"].sum * 100.0 / meters["total"].sum, 3 - ) - if meters["total"].sum > 0 - else float("nan"), - ) - - @staticmethod - def logging_outputs_can_be_summed() -> bool: - """ - Whether the logging outputs returned by `forward` can be summed - across workers prior to calling `reduce_metrics`. Setting this - to True will improves distributed training speed. - """ - return True diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/label_smoothed_cross_entropy_latency_augmented.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/label_smoothed_cross_entropy_latency_augmented.py deleted file mode 100644 index 606da98c51f444a7d72e08f950776e60c1fe1202..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/label_smoothed_cross_entropy_latency_augmented.py +++ /dev/null @@ -1,221 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from dataclasses import dataclass, field - -import torch -from fairseq import metrics, utils -from fairseq.criterions import register_criterion -from fairseq.criterions.label_smoothed_cross_entropy import ( - LabelSmoothedCrossEntropyCriterion, - LabelSmoothedCrossEntropyCriterionConfig, -) - -try: - from simuleval.metrics.latency import ( - AverageLagging, - AverageProportion, - DifferentiableAverageLagging, - ) - - LATENCY_METRICS = { - "average_lagging": AverageLagging, - "average_proportion": AverageProportion, - "differentiable_average_lagging": DifferentiableAverageLagging, - } -except ImportError: - LATENCY_METRICS = None - - -@dataclass -class LabelSmoothedCrossEntropyCriterionLatencyAugmentConfig( - LabelSmoothedCrossEntropyCriterionConfig -): - latency_avg_weight: float = field( - default=0.0, - metadata={"help": "weight fot average latency loss."}, - ) - latency_var_weight: float = field( - default=0.0, - metadata={"help": "weight fot variance latency loss."}, - ) - latency_avg_type: str = field( - default="differentiable_average_lagging", - metadata={"help": "latency type for average loss"}, - ) - latency_var_type: str = field( - default="variance_delay", - metadata={"help": "latency typ for variance loss"}, - ) - latency_gather_method: str = field( - default="weighted_average", - metadata={"help": "method to gather latency loss for all heads"}, - ) - latency_update_after: int = field( - default=0, - metadata={"help": "Add latency loss after certain steps"}, - ) - - -@register_criterion( - "latency_augmented_label_smoothed_cross_entropy", - dataclass=LabelSmoothedCrossEntropyCriterionLatencyAugmentConfig, -) -class LatencyAugmentedLabelSmoothedCrossEntropyCriterion( - LabelSmoothedCrossEntropyCriterion -): - def __init__( - self, - task, - sentence_avg, - label_smoothing, - ignore_prefix_size, - report_accuracy, - latency_avg_weight, - latency_var_weight, - latency_avg_type, - latency_var_type, - latency_gather_method, - latency_update_after, - ): - super().__init__( - task, sentence_avg, label_smoothing, ignore_prefix_size, report_accuracy - ) - assert LATENCY_METRICS is not None, "Please make sure SimulEval is installed." - - self.latency_avg_weight = latency_avg_weight - self.latency_var_weight = latency_var_weight - self.latency_avg_type = latency_avg_type - self.latency_var_type = latency_var_type - self.latency_gather_method = latency_gather_method - self.latency_update_after = latency_update_after - - def forward(self, model, sample, reduce=True): - net_output = model(**sample["net_input"]) - # 1. Compute cross entropy loss - loss, nll_loss = self.compute_loss(model, net_output, sample, reduce=reduce) - - # 2. Compute cross latency loss - latency_loss, expected_latency, expected_delays_var = self.compute_latency_loss( - model, sample, net_output - ) - - if self.latency_update_after > 0: - num_updates = getattr(model.decoder, "num_updates", None) - assert ( - num_updates is not None - ), "model.decoder doesn't have attribute 'num_updates'" - if num_updates <= self.latency_update_after: - latency_loss = 0 - - loss += latency_loss - - sample_size = ( - sample["target"].size(0) if self.sentence_avg else sample["ntokens"] - ) - - logging_output = { - "loss": loss.data, - "nll_loss": nll_loss.data, - "ntokens": sample["ntokens"], - "nsentences": sample["target"].size(0), - "sample_size": sample_size, - "latency": expected_latency, - "delays_var": expected_delays_var, - "latency_loss": latency_loss, - } - - if self.report_accuracy: - n_correct, total = self.compute_accuracy(model, net_output, sample) - logging_output["n_correct"] = utils.item(n_correct.data) - logging_output["total"] = utils.item(total.data) - return loss, sample_size, logging_output - - def compute_latency_loss(self, model, sample, net_output): - assert ( - net_output[-1].encoder_padding_mask is None - or not net_output[-1].encoder_padding_mask[:, 0].any() - ), "Only right padding on source is supported." - # 1. Obtain the expected alignment - alpha_list = [item["alpha"] for item in net_output[1].attn_list] - num_layers = len(alpha_list) - bsz, num_heads, tgt_len, src_len = alpha_list[0].size() - - # bsz * num_layers * num_heads, tgt_len, src_len - alpha_all = torch.cat(alpha_list, dim=1).view(-1, tgt_len, src_len) - - # 2 compute expected delays - # bsz * num_heads * num_layers, tgt_len, src_len for MMA - steps = ( - torch.arange(1, 1 + src_len) - .unsqueeze(0) - .unsqueeze(1) - .expand_as(alpha_all) - .type_as(alpha_all) - ) - - expected_delays = torch.sum(steps * alpha_all, dim=-1) - - target_padding_mask = ( - model.get_targets(sample, net_output) - .eq(self.padding_idx) - .unsqueeze(1) - .expand(bsz, num_layers * num_heads, tgt_len) - .contiguous() - .view(-1, tgt_len) - ) - - src_lengths = ( - sample["net_input"]["src_lengths"] - .unsqueeze(1) - .expand(bsz, num_layers * num_heads) - .contiguous() - .view(-1) - ) - expected_latency = LATENCY_METRICS[self.latency_avg_type]( - expected_delays, src_lengths, None, target_padding_mask=target_padding_mask - ) - - # 2.1 average expected latency of heads - # bsz, num_layers * num_heads - expected_latency = expected_latency.view(bsz, -1) - if self.latency_gather_method == "average": - # bsz * tgt_len - expected_latency = expected_delays.mean(dim=1) - elif self.latency_gather_method == "weighted_average": - weights = torch.nn.functional.softmax(expected_latency, dim=1) - expected_latency = torch.sum(expected_latency * weights, dim=1) - elif self.latency_gather_method == "max": - expected_latency = expected_latency.max(dim=1)[0] - else: - raise NotImplementedError - - expected_latency = expected_latency.sum() - avg_loss = self.latency_avg_weight * expected_latency - - # 2.2 variance of expected delays - expected_delays_var = ( - expected_delays.view(bsz, -1, tgt_len).var(dim=1).mean(dim=1) - ) - expected_delays_var = expected_delays_var.sum() - var_loss = self.latency_avg_weight * expected_delays_var - - # 3. Final loss - latency_loss = avg_loss + var_loss - - return latency_loss, expected_latency, expected_delays_var - - @classmethod - def reduce_metrics(cls, logging_outputs) -> None: - super().reduce_metrics(logging_outputs) - latency = sum(log.get("latency", 0) for log in logging_outputs) - delays_var = sum(log.get("delays_var", 0) for log in logging_outputs) - latency_loss = sum(log.get("latency_loss", 0) for log in logging_outputs) - nsentences = sum(log.get("nsentences", 0) for log in logging_outputs) - metrics.log_scalar("latency", latency.float() / nsentences, nsentences, round=3) - metrics.log_scalar("delays_var", delays_var / nsentences, nsentences, round=3) - metrics.log_scalar( - "latency_loss", latency_loss / nsentences, nsentences, round=3 - ) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/label_smoothed_cross_entropy_with_alignment.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/label_smoothed_cross_entropy_with_alignment.py deleted file mode 100644 index d855ab78966ad528d2cae6fb21797bcfc94905bd..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/label_smoothed_cross_entropy_with_alignment.py +++ /dev/null @@ -1,129 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import math -from dataclasses import dataclass, field - -from fairseq import metrics, utils -from fairseq.criterions import register_criterion - -from .label_smoothed_cross_entropy import ( - LabelSmoothedCrossEntropyCriterion, - LabelSmoothedCrossEntropyCriterionConfig, -) - - -@dataclass -class LabelSmoothedCrossEntropyCriterionWithAlignmentConfig( - LabelSmoothedCrossEntropyCriterionConfig -): - alignment_lambda: float = field( - default=0.05, metadata={"help": "weight for the alignment loss"} - ) - - -@register_criterion( - "label_smoothed_cross_entropy_with_alignment", - dataclass=LabelSmoothedCrossEntropyCriterionWithAlignmentConfig, -) -class LabelSmoothedCrossEntropyCriterionWithAlignment( - LabelSmoothedCrossEntropyCriterion -): - def __init__(self, task, sentence_avg, label_smoothing, alignment_lambda): - super().__init__(task, sentence_avg, label_smoothing) - self.alignment_lambda = alignment_lambda - - def forward(self, model, sample, reduce=True): - """Compute the loss for the given sample. - - Returns a tuple with three elements: - 1) the loss - 2) the sample size, which is used as the denominator for the gradient - 3) logging outputs to display while training - """ - net_output = model(**sample["net_input"]) - loss, nll_loss = self.compute_loss(model, net_output, sample, reduce=reduce) - sample_size = ( - sample["target"].size(0) if self.sentence_avg else sample["ntokens"] - ) - logging_output = { - "loss": utils.item(loss.data) if reduce else loss.data, - "nll_loss": utils.item(nll_loss.data) if reduce else nll_loss.data, - "ntokens": sample["ntokens"], - "nsentences": sample["target"].size(0), - "sample_size": sample_size, - } - - alignment_loss = None - - # Compute alignment loss only for training set and non dummy batches. - if "alignments" in sample and sample["alignments"] is not None: - alignment_loss = self.compute_alignment_loss(sample, net_output) - - if alignment_loss is not None: - logging_output["alignment_loss"] = utils.item(alignment_loss.data) - loss += self.alignment_lambda * alignment_loss - - return loss, sample_size, logging_output - - def compute_alignment_loss(self, sample, net_output): - attn_prob = net_output[1]["attn"][0] - bsz, tgt_sz, src_sz = attn_prob.shape - attn = attn_prob.view(bsz * tgt_sz, src_sz) - - align = sample["alignments"] - align_weights = sample["align_weights"].float() - - if len(align) > 0: - # Alignment loss computation. align (shape [:, 2]) contains the src-tgt index pairs corresponding to - # the alignments. align_weights (shape [:]) contains the 1 / frequency of a tgt index for normalizing. - loss = -( - (attn[align[:, 1][:, None], align[:, 0][:, None]]).log() - * align_weights[:, None] - ).sum() - else: - return None - - return loss - - @staticmethod - def reduce_metrics(logging_outputs) -> None: - """Aggregate logging outputs from data parallel training.""" - loss_sum = utils.item(sum(log.get("loss", 0) for log in logging_outputs)) - nll_loss_sum = utils.item( - sum(log.get("nll_loss", 0) for log in logging_outputs) - ) - alignment_loss_sum = utils.item( - sum(log.get("alignment_loss", 0) for log in logging_outputs) - ) - ntokens = utils.item(sum(log.get("ntokens", 0) for log in logging_outputs)) - sample_size = utils.item( - sum(log.get("sample_size", 0) for log in logging_outputs) - ) - - metrics.log_scalar( - "loss", loss_sum / sample_size / math.log(2), sample_size, round=3 - ) - metrics.log_scalar( - "nll_loss", nll_loss_sum / ntokens / math.log(2), ntokens, round=3 - ) - metrics.log_scalar( - "alignment_loss", - alignment_loss_sum / sample_size / math.log(2), - sample_size, - round=3, - ) - metrics.log_derived( - "ppl", lambda meters: utils.get_perplexity(meters["nll_loss"].avg) - ) - - @staticmethod - def logging_outputs_can_be_summed() -> bool: - """ - Whether the logging outputs returned by `forward` can be summed - across workers prior to calling `reduce_metrics`. Setting this - to True will improves distributed training speed. - """ - return True diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/label_smoothed_cross_entropy_with_ctc.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/label_smoothed_cross_entropy_with_ctc.py deleted file mode 100644 index b696c80b8a79daaaa1fd4d06fccc5f707db2ee7a..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/label_smoothed_cross_entropy_with_ctc.py +++ /dev/null @@ -1,95 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import math -from dataclasses import dataclass, field - -import torch -import torch.nn.functional as F -from fairseq import metrics, utils -from fairseq.criterions import register_criterion -from fairseq.criterions.label_smoothed_cross_entropy import ( - LabelSmoothedCrossEntropyCriterion, - LabelSmoothedCrossEntropyCriterionConfig, -) -from fairseq.data.data_utils import lengths_to_mask - - -@dataclass -class LabelSmoothedCrossEntropyWithCtcCriterionConfig( - LabelSmoothedCrossEntropyCriterionConfig -): - ctc_weight: float = field(default=1.0, metadata={"help": "weight for CTC loss"}) - - -@register_criterion( - "label_smoothed_cross_entropy_with_ctc", - dataclass=LabelSmoothedCrossEntropyWithCtcCriterionConfig, -) -class LabelSmoothedCrossEntropyWithCtcCriterion(LabelSmoothedCrossEntropyCriterion): - def __init__( - self, - task, - sentence_avg, - label_smoothing, - ignore_prefix_size, - report_accuracy, - ctc_weight, - ): - super().__init__( - task, sentence_avg, label_smoothing, ignore_prefix_size, report_accuracy - ) - self.ctc_weight = ctc_weight - - def forward(self, model, sample, reduce=True): - net_output = model(**sample["net_input"]) - loss, nll_loss = self.compute_loss(model, net_output, sample, reduce=reduce) - - ctc_loss = torch.tensor(0.0).type_as(loss) - if self.ctc_weight > 0.0: - ctc_lprobs, ctc_lens = model.get_ctc_output(net_output, sample) - ctc_tgt, ctc_tgt_lens = model.get_ctc_target(sample) - ctc_tgt_mask = lengths_to_mask(ctc_tgt_lens) - ctc_tgt_flat = ctc_tgt.masked_select(ctc_tgt_mask) - reduction = "sum" if reduce else "none" - ctc_loss = ( - F.ctc_loss( - ctc_lprobs, - ctc_tgt_flat, - ctc_lens, - ctc_tgt_lens, - reduction=reduction, - zero_infinity=True, - ) - * self.ctc_weight - ) - loss += ctc_loss - - sample_size = ( - sample["target"].size(0) if self.sentence_avg else sample["ntokens"] - ) - logging_output = { - "loss": utils.item(loss.data), - "nll_loss": utils.item(nll_loss.data), - "ctc_loss": utils.item(ctc_loss.data), - "ntokens": sample["ntokens"], - "nsentences": sample["target"].size(0), - "sample_size": sample_size, - } - if self.report_accuracy: - n_correct, total = self.compute_accuracy(model, net_output, sample) - logging_output["n_correct"] = utils.item(n_correct.data) - logging_output["total"] = utils.item(total.data) - return loss, sample_size, logging_output - - @classmethod - def reduce_metrics(cls, logging_outputs) -> None: - super().reduce_metrics(logging_outputs) - loss_sum = sum(log.get("ctc_loss", 0) for log in logging_outputs) - sample_size = sum(log.get("sample_size", 0) for log in logging_outputs) - - metrics.log_scalar( - "ctc_loss", loss_sum / sample_size / math.log(2), sample_size, round=3 - ) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/legacy_masked_lm.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/legacy_masked_lm.py deleted file mode 100644 index c70608c5a143b7b4fbd8c58dfcf9f873639d379c..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/legacy_masked_lm.py +++ /dev/null @@ -1,177 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import math - -import torch -import torch.nn.functional as F -from fairseq import metrics, utils -from fairseq.criterions import FairseqCriterion, register_criterion - - -def compute_cross_entropy_loss(logits, targets, ignore_index=-100): - """ - Function to compute the cross entropy loss. The default value of - ignore_index is the same as the default value for F.cross_entropy in - pytorch. - """ - assert logits.size(0) == targets.size( - -1 - ), "Logits and Targets tensor shapes don't match up" - - loss = F.nll_loss( - F.log_softmax(logits, -1, dtype=torch.float32), - targets, - reduction="sum", - ignore_index=ignore_index, - ) - return loss - - -@register_criterion("legacy_masked_lm_loss") -class LegacyMaskedLmLoss(FairseqCriterion): - """ - Implementation for the loss used in masked language model (MLM) training. - This optionally also computes the next sentence prediction (NSP) loss and - adds it to the overall loss based on the specified args. There are three - cases to consider: - 1) Generic MLM training without NSP loss. In this case sentence_targets - and sentence_logits are both None. - 2) BERT training without NSP loss. In this case sentence_targets is - not None but sentence_logits is None and we should not be computing - a sentence level loss. - 3) BERT training with NSP loss. In this case both sentence_targets and - sentence_logits are not None and we should be computing a sentence - level loss. The weight of the sentence level loss is specified as - an argument. - """ - - def __init__(self, task, masked_lm_only, nsp_loss_weight): - super().__init__(task) - self.masked_lm_only = masked_lm_only - self.nsp_loss_weight = nsp_loss_weight - - @staticmethod - def add_args(parser): - """Args for MaskedLM Loss""" - # Default for masked_lm_only is False so as to not break BERT training - parser.add_argument( - "--masked-lm-only", - default=False, - action="store_true", - help="compute MLM loss only", - ) - parser.add_argument( - "--nsp-loss-weight", - default=1.0, - type=float, - help="weight for next sentence prediction" " loss (default 1)", - ) - - def forward(self, model, sample, reduce=True): - """Compute the loss for the given sample. - Returns a tuple with three elements: - 1) the loss - 2) the sample size, which is used as the denominator for the gradient - 3) logging outputs to display while training - """ - lm_logits, output_metadata = model(**sample["net_input"]) - - # reshape lm_logits from (N,T,C) to (N*T,C) - lm_logits = lm_logits.view(-1, lm_logits.size(-1)) - lm_targets = sample["lm_target"].view(-1) - lm_loss = compute_cross_entropy_loss(lm_logits, lm_targets, self.padding_idx) - - # compute the number of tokens for which loss is computed. This is used - # to normalize the loss - ntokens = utils.strip_pad(lm_targets, self.padding_idx).numel() - loss = lm_loss / ntokens - nsentences = sample["nsentences"] - # nsentences = 0 - - # Compute sentence loss if masked_lm_only is False - sentence_loss = None - if not self.masked_lm_only: - sentence_logits = output_metadata["sentence_logits"] - sentence_targets = sample["sentence_target"].view(-1) - # This needs to be recomputed due to some differences between - # TokenBlock and BlockPair dataset. This can be resolved with a - # refactor of BERTModel which we will do in the future. - # TODO: Remove this after refactor of BERTModel - nsentences = sentence_targets.size(0) - - # Check for logits being none which can happen when remove_heads - # is set to true in the BERT model. Ideally we should set - # masked_lm_only to true in this case, but that requires some - # refactor in the BERT model. - if sentence_logits is not None: - sentence_loss = compute_cross_entropy_loss( - sentence_logits, sentence_targets - ) - - loss += self.nsp_loss_weight * (sentence_loss / nsentences) - - # NOTE: as we are summing up per token mlm loss and per sentence nsp loss - # we don't need to use sample_size as denominator for the gradient - # here sample_size is just used for logging - sample_size = 1 - logging_output = { - "loss": utils.item(loss.data) if reduce else loss.data, - "lm_loss": utils.item(lm_loss.data) if reduce else lm_loss.data, - # sentence loss is not always computed - "sentence_loss": ( - (utils.item(sentence_loss.data) if reduce else sentence_loss.data) - if sentence_loss is not None - else 0.0 - ), - "ntokens": ntokens, - "nsentences": nsentences, - "sample_size": sample_size, - } - return loss, sample_size, logging_output - - @staticmethod - def reduce_metrics(logging_outputs) -> None: - """Aggregate logging outputs from data parallel training.""" - lm_loss_sum = sum(log.get("lm_loss", 0) for log in logging_outputs) - sentence_loss_sum = sum(log.get("sentence_loss", 0) for log in logging_outputs) - ntokens = sum(log.get("ntokens", 0) for log in logging_outputs) - nsentences = sum(log.get("nsentences", 0) for log in logging_outputs) - sample_size = sum(log.get("sample_size", 0) for log in logging_outputs) - agg_loss = sum(log.get("loss", 0) for log in logging_outputs) - - metrics.log_scalar( - "loss", - agg_loss / sample_size / math.log(2) if sample_size > 0 else 0.0, - sample_size, - round=3, - ) - metrics.log_scalar( - "lm_loss", - lm_loss_sum / ntokens / math.log(2) if ntokens > 0 else 0.0, - ntokens, - round=3, - ) - metrics.log_scalar( - "sentence_loss", - sentence_loss_sum / nsentences / math.log(2) if nsentences > 0 else 0.0, - nsentences, - round=3, - ) - metrics.log_scalar( - "nll_loss", - lm_loss_sum / ntokens / math.log(2) if ntokens > 0 else 0.0, - ntokens, - round=3, - ) - - @staticmethod - def logging_outputs_can_be_summed() -> bool: - """ - Whether the logging outputs returned by `forward` can be summed - across workers prior to calling `reduce_metrics`. Setting this - to True will improves distributed training speed. - """ - return True diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/masked_lm.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/masked_lm.py deleted file mode 100644 index 40e39f4cc3b3047766fa134afa4d8149a39f283a..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/masked_lm.py +++ /dev/null @@ -1,98 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import math -from dataclasses import dataclass - -import torch -from fairseq import metrics, modules, utils -from fairseq.criterions import FairseqCriterion, register_criterion -from fairseq.dataclass import FairseqDataclass -from omegaconf import II - - -@dataclass -class MaskedLmConfig(FairseqDataclass): - tpu: bool = II("common.tpu") - - -@register_criterion("masked_lm", dataclass=MaskedLmConfig) -class MaskedLmLoss(FairseqCriterion): - """ - Implementation for the loss used in masked language model (MLM) training. - """ - - def __init__(self, cfg: MaskedLmConfig, task): - super().__init__(task) - self.tpu = cfg.tpu - - def forward(self, model, sample, reduce=True): - """Compute the loss for the given sample. - - Returns a tuple with three elements: - 1) the loss - 2) the sample size, which is used as the denominator for the gradient - 3) logging outputs to display while training - """ - masked_tokens = sample["target"].ne(self.padding_idx) - sample_size = masked_tokens.int().sum() - - # Rare: when all tokens are masked, project all tokens. - # We use torch.where to avoid device-to-host transfers, - # except on CPU where torch.where is not well supported - # (see github.com/pytorch/pytorch/issues/26247). - if self.tpu: - masked_tokens = None # always project all tokens on TPU - elif masked_tokens.device == torch.device("cpu"): - if not masked_tokens.any(): - masked_tokens = None - else: - masked_tokens = torch.where( - masked_tokens.any(), - masked_tokens, - masked_tokens.new([True]), - ) - - logits = model(**sample["net_input"], masked_tokens=masked_tokens)[0] - targets = model.get_targets(sample, [logits]) - if masked_tokens is not None: - targets = targets[masked_tokens] - - loss = modules.cross_entropy( - logits.view(-1, logits.size(-1)), - targets.view(-1), - reduction="sum", - ignore_index=self.padding_idx, - ) - - logging_output = { - "loss": loss if self.tpu else loss.data, - "ntokens": sample["ntokens"], - "nsentences": sample["nsentences"], - "sample_size": sample_size, - } - return loss, sample_size, logging_output - - @staticmethod - def reduce_metrics(logging_outputs) -> None: - """Aggregate logging outputs from data parallel training.""" - loss_sum = sum(log.get("loss", 0) for log in logging_outputs) - sample_size = sum(log.get("sample_size", 0) for log in logging_outputs) - - metrics.log_scalar( - "loss", loss_sum / sample_size / math.log(2), sample_size, round=3 - ) - metrics.log_derived( - "ppl", lambda meters: utils.get_perplexity(meters["loss"].avg) - ) - - @staticmethod - def logging_outputs_can_be_summed() -> bool: - """ - Whether the logging outputs returned by `forward` can be summed - across workers prior to calling `reduce_metrics`. Setting this - to True will improves distributed training speed. - """ - return True diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/model_criterion.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/model_criterion.py deleted file mode 100644 index 03321de222242b1c55e9e1d452c93071dfe7d3c0..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/model_criterion.py +++ /dev/null @@ -1,153 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import logging -from dataclasses import dataclass, field -from typing import Dict, List - -import torch -from fairseq import metrics, utils -from fairseq.criterions import FairseqCriterion, register_criterion -from fairseq.dataclass import FairseqDataclass - -logger = logging.getLogger(__name__) - - -@dataclass -class ModelCriterionConfig(FairseqDataclass): - loss_weights: Dict[str, float] = field( - default_factory=dict, - metadata={"help": "weights for the loss terms"}, - ) - log_keys: List[str] = field( - default_factory=list, - metadata={"help": "additional output keys to log"}, - ) - - -@register_criterion("model", dataclass=ModelCriterionConfig) -class ModelCriterion(FairseqCriterion): - """ - This criterion relies on the model to supply losses. - The losses should be a dictionary of name -> scalar returned by - the model either by including it in the net_output dict or by - implementing a get_losses(net_output, sample) method. The final loss is - a scaled sum of all losses according to weights in loss_weights. - If no weights are provided, then all losses are scaled by 1.0. - - The losses will be automatically logged. Additional keys from - net_output dict can be logged via the log_keys parameter. - """ - - def __init__(self, task, loss_weights=None, log_keys=None): - super().__init__(task) - self.loss_weights = loss_weights - self.log_keys = log_keys - - def forward(self, model, sample, reduce=True): - net_output = model(**sample["net_input"]) - - scaled_losses = {} - - if hasattr(model, "get_losses"): - losses = model.get_losses(net_output, sample) - elif isinstance(net_output, dict) and "losses" in net_output: - losses = net_output["losses"] - else: - raise Exception("Could not retrieve losses") - - for lk, p in losses.items(): - try: - coef = 1.0 if len(self.loss_weights) == 0 else self.loss_weights[lk] - except KeyError: - logger.error( - f"weight for loss {lk} is not in loss_weights ({self.loss_weights})" - ) - raise - if coef != 0 and p is not None: - scaled_losses[lk] = coef * p.float() - - loss = sum(scaled_losses.values()) - - if "sample_size" in net_output: - sample_size = net_output["sample_size"] - else: - sample_size = loss.numel() - - if reduce and loss.numel() > 1: - loss = loss.sum() - - logging_output = { - "loss": loss.data, - "ntokens": sample_size, - "nsentences": sample["id"].numel(), - "sample_size": sample_size, - "_world_size": 1, - } - - for lk in self.log_keys: - if lk in net_output and net_output[lk] is not None: - if not torch.is_tensor(net_output[lk]) or net_output[lk].numel() == 1: - logging_output[lk] = float(net_output[lk]) - else: - for i, v in enumerate(net_output[lk]): - logging_output[f"{lk}_{i}"] = float(v) - - if len(scaled_losses) > 1: - for lk, l in scaled_losses.items(): - if l.numel() > 1: - l = l.sum() - logging_output[f"loss_{lk}"] = l.item() - - if "logs" in net_output: - for lgw in net_output["logs"]: - logging_output[lgw] = net_output["logs"][lgw] - - return loss, sample_size, logging_output - - @staticmethod - def reduce_metrics(logging_outputs) -> None: - """Aggregate logging outputs from data parallel training.""" - loss_sum = utils.item(sum(log.get("loss", 0) for log in logging_outputs)) - ntokens = utils.item(sum(log.get("ntokens", 0) for log in logging_outputs)) - nsentences = utils.item( - sum(log.get("nsentences", 0) for log in logging_outputs) - ) - sample_size = utils.item( - sum(log.get("sample_size", 0) for log in logging_outputs) - ) - - metrics.log_scalar("loss", loss_sum / sample_size, sample_size, round=3) - metrics.log_scalar("ntokens", ntokens) - metrics.log_scalar("nsentences", nsentences) - - builtin_keys = { - "loss", - "ntokens", - "nsentences", - "sample_size", - "_world_size", - } - - world_size = utils.item( - sum(log.get("_world_size", 0) for log in logging_outputs) - ) - - for k in logging_outputs[0]: - if k not in builtin_keys: - val = sum(log.get(k, 0) for log in logging_outputs) - if k.startswith("loss_"): - metrics.log_scalar(k, val / sample_size, sample_size, round=3) - else: - metrics.log_scalar(k, val / world_size, round=3) - - @staticmethod - def logging_outputs_can_be_summed() -> bool: - """ - Whether the logging outputs returned by `forward` can be summed - across workers prior to calling `reduce_metrics`. Setting this - to True will improves distributed training speed. - """ - return True diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/nat_loss.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/nat_loss.py deleted file mode 100644 index 2f0dc7649dd3d5246b49f64ae91b5a15eb3f14de..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/nat_loss.py +++ /dev/null @@ -1,179 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import math -from dataclasses import dataclass, field - -import torch -import torch.nn.functional as F -from fairseq import metrics, utils -from fairseq.criterions import FairseqCriterion, register_criterion -from fairseq.dataclass import FairseqDataclass -from torch import Tensor - - -@dataclass -class LabelSmoothedDualImitationCriterionConfig(FairseqDataclass): - label_smoothing: float = field( - default=0.0, - metadata={"help": "epsilon for label smoothing, 0 means no label smoothing"}, - ) - - -@register_criterion("nat_loss", dataclass=LabelSmoothedDualImitationCriterionConfig) -class LabelSmoothedDualImitationCriterion(FairseqCriterion): - def __init__(self, task, label_smoothing): - super().__init__(task) - self.label_smoothing = label_smoothing - - def _compute_loss( - self, outputs, targets, masks=None, label_smoothing=0.0, name="loss", factor=1.0 - ): - """ - outputs: batch x len x d_model - targets: batch x len - masks: batch x len - - policy_logprob: if there is some policy - depends on the likelihood score as rewards. - """ - - def mean_ds(x: Tensor, dim=None) -> Tensor: - return ( - x.float().mean().type_as(x) - if dim is None - else x.float().mean(dim).type_as(x) - ) - - if masks is not None: - outputs, targets = outputs[masks], targets[masks] - - if masks is not None and not masks.any(): - nll_loss = torch.tensor(0) - loss = nll_loss - else: - logits = F.log_softmax(outputs, dim=-1) - if targets.dim() == 1: - losses = F.nll_loss(logits, targets.to(logits.device), reduction="none") - - else: # soft-labels - losses = F.kl_div(logits, targets.to(logits.device), reduction="none") - losses = losses.sum(-1) - - nll_loss = mean_ds(losses) - if label_smoothing > 0: - loss = ( - nll_loss * (1 - label_smoothing) - mean_ds(logits) * label_smoothing - ) - else: - loss = nll_loss - - loss = loss * factor - return {"name": name, "loss": loss, "nll_loss": nll_loss, "factor": factor} - - def _custom_loss(self, loss, name="loss", factor=1.0): - return {"name": name, "loss": loss, "factor": factor} - - def forward(self, model, sample, reduce=True): - """Compute the loss for the given sample. - Returns a tuple with three elements: - 1) the loss - 2) the sample size, which is used as the denominator for the gradient - 3) logging outputs to display while training - """ - nsentences, ntokens = sample["nsentences"], sample["ntokens"] - - # B x T - src_tokens, src_lengths = ( - sample["net_input"]["src_tokens"], - sample["net_input"]["src_lengths"], - ) - tgt_tokens, prev_output_tokens = sample["target"], sample["prev_target"] - - outputs = model(src_tokens, src_lengths, prev_output_tokens, tgt_tokens) - losses, nll_loss = [], [] - - for obj in outputs: - if outputs[obj].get("loss", None) is None: - _losses = self._compute_loss( - outputs[obj].get("out"), - outputs[obj].get("tgt"), - outputs[obj].get("mask", None), - outputs[obj].get("ls", 0.0), - name=obj + "-loss", - factor=outputs[obj].get("factor", 1.0), - ) - else: - _losses = self._custom_loss( - outputs[obj].get("loss"), - name=obj + "-loss", - factor=outputs[obj].get("factor", 1.0), - ) - - losses += [_losses] - if outputs[obj].get("nll_loss", False): - nll_loss += [_losses.get("nll_loss", 0.0)] - - loss = sum(l["loss"] for l in losses) - nll_loss = sum(l for l in nll_loss) if len(nll_loss) > 0 else loss.new_tensor(0) - - # NOTE: - # we don't need to use sample_size as denominator for the gradient - # here sample_size is just used for logging - sample_size = 1 - logging_output = { - "loss": loss.data, - "nll_loss": nll_loss.data, - "ntokens": ntokens, - "nsentences": nsentences, - "sample_size": sample_size, - } - - for l in losses: - logging_output[l["name"]] = ( - utils.item(l["loss"].data / l["factor"]) - if reduce - else l[["loss"]].data / l["factor"] - ) - - return loss, sample_size, logging_output - - @staticmethod - def reduce_metrics(logging_outputs) -> None: - """Aggregate logging outputs from data parallel training.""" - sample_size = utils.item( - sum(log.get("sample_size", 0) for log in logging_outputs) - ) - loss = utils.item(sum(log.get("loss", 0) for log in logging_outputs)) - nll_loss = utils.item(sum(log.get("nll_loss", 0) for log in logging_outputs)) - - metrics.log_scalar( - "loss", loss / sample_size / math.log(2), sample_size, round=3 - ) - metrics.log_scalar( - "nll_loss", nll_loss / sample_size / math.log(2), sample_size, round=3 - ) - metrics.log_derived( - "ppl", lambda meters: utils.get_perplexity(meters["loss"].avg) - ) - - for key in logging_outputs[0]: - if key[-5:] == "-loss": - val = sum(log.get(key, 0) for log in logging_outputs) - metrics.log_scalar( - key[:-5], - val / sample_size / math.log(2) if sample_size > 0 else 0.0, - sample_size, - round=3, - ) - - @staticmethod - def logging_outputs_can_be_summed() -> bool: - """ - Whether the logging outputs returned by `forward` can be summed - across workers prior to calling `reduce_metrics`. Setting this - to True will improves distributed training speed. - """ - return True diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/sentence_prediction.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/sentence_prediction.py deleted file mode 100644 index b8f98f814270662b9c7a4a08be0ea73f2a4e033f..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/sentence_prediction.py +++ /dev/null @@ -1,140 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import math -from dataclasses import dataclass, field - -import torch -import torch.nn.functional as F -from fairseq import metrics -from fairseq.criterions import FairseqCriterion, register_criterion -from fairseq.dataclass import FairseqDataclass - - -@dataclass -class SentencePredictionConfig(FairseqDataclass): - classification_head_name: str = field( - default="sentence_classification_head", - metadata={"help": "name of the classification head to use"}, - ) - regression_target: bool = field( - default=False, - ) - - -@register_criterion("sentence_prediction", dataclass=SentencePredictionConfig) -class SentencePredictionCriterion(FairseqCriterion): - def __init__(self, cfg: SentencePredictionConfig, task): - super().__init__(task) - self.classification_head_name = cfg.classification_head_name - self.regression_target = cfg.regression_target - - def forward(self, model, sample, reduce=True): - """Compute the loss for the given sample. - - Returns a tuple with three elements: - 1) the loss - 2) the sample size, which is used as the denominator for the gradient - 3) logging outputs to display while training - """ - assert ( - hasattr(model, "classification_heads") - and self.classification_head_name in model.classification_heads - ), "model must provide sentence classification head for --criterion=sentence_prediction" - - logits, _ = model( - **sample["net_input"], - features_only=True, - classification_head_name=self.classification_head_name, - ) - targets = model.get_targets(sample, [logits]).view(-1) - sample_size = targets.numel() - - if not self.regression_target: - lprobs = F.log_softmax(logits, dim=-1, dtype=torch.float32) - task_loss = F.nll_loss(lprobs, targets, reduction="sum") - else: - logits = logits.view(-1).float() - targets = targets.float() - task_loss = F.mse_loss(logits, targets, reduction="sum") - - logging_output = {} - loss = task_loss - # mha & ffn regularization update - if ( - hasattr(model.args, "mha_reg_scale_factor") - and model.args.mha_reg_scale_factor != 0.0 - ): - mha_reg_loss = model._get_adaptive_head_loss() - loss += mha_reg_loss - logging_output.update({"mha_reg_loss": mha_reg_loss}) - if ( - hasattr(model.args, "ffn_reg_scale_factor") - and model.args.ffn_reg_scale_factor != 0.0 - ): - ffn_reg_loss = model._get_adaptive_ffn_loss() - loss += ffn_reg_loss - logging_output.update({"ffn_reg_loss": ffn_reg_loss}) - - logging_output.update( - { - "loss": loss.data, - "ntokens": sample["ntokens"], - "nsentences": sample_size, - "sample_size": sample_size, - } - ) - if not self.regression_target: - preds = logits.argmax(dim=1) - logging_output["ncorrect"] = (preds == targets).sum() - - return loss, sample_size, logging_output - - @staticmethod - def reduce_metrics(logging_outputs) -> None: - """Aggregate logging outputs from data parallel training.""" - loss_sum = sum(log.get("loss", 0) for log in logging_outputs) - ntokens = sum(log.get("ntokens", 0) for log in logging_outputs) - nsentences = sum(log.get("nsentences", 0) for log in logging_outputs) - sample_size = sum(log.get("sample_size", 0) for log in logging_outputs) - mha_reg_loss_sum = sum(log.get("mha_reg_loss", 0) for log in logging_outputs) - ffn_reg_loss_sum = sum(log.get("ffn_reg_loss", 0) for log in logging_outputs) - - metrics.log_scalar( - "loss", loss_sum / sample_size / math.log(2), sample_size, round=3 - ) - if mha_reg_loss_sum: - metrics.log_scalar( - "mha_reg_loss", - mha_reg_loss_sum / sample_size / math.log(2), - sample_size, - round=3, - ) - if ffn_reg_loss_sum: - metrics.log_scalar( - "ffn_reg_loss", - ffn_reg_loss_sum / sample_size / math.log(2), - sample_size, - round=3, - ) - if sample_size != ntokens: - metrics.log_scalar( - "nll_loss", loss_sum / ntokens / math.log(2), ntokens, round=3 - ) - - if len(logging_outputs) > 0 and "ncorrect" in logging_outputs[0]: - ncorrect = sum(log.get("ncorrect", 0) for log in logging_outputs) - metrics.log_scalar( - "accuracy", 100.0 * ncorrect / nsentences, nsentences, round=1 - ) - - @staticmethod - def logging_outputs_can_be_summed() -> bool: - """ - Whether the logging outputs returned by `forward` can be summed - across workers prior to calling `reduce_metrics`. Setting this - to True will improves distributed training speed. - """ - return True diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/sentence_prediction_adapters.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/sentence_prediction_adapters.py deleted file mode 100644 index 37258ce8f6647c2558788f13ea404442f03fef75..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/sentence_prediction_adapters.py +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import torch -import torch.nn.functional as F -from fairseq.criterions import register_criterion -from fairseq.criterions.sentence_prediction import ( - SentencePredictionConfig, - SentencePredictionCriterion, -) - - -@register_criterion("sentence_prediction_adapters", dataclass=SentencePredictionConfig) -class SentencePredictionCriterionAdapters(SentencePredictionCriterion): - def forward(self, model, sample, reduce=True): - """Compute the loss for the given sample. - - Returns a tuple with three elements: - 1) the loss - 2) the sample size, which is used as the denominator for the gradient - 3) logging outputs to display while training - """ - assert ( - hasattr(model, "classification_heads") - and self.classification_head_name in model.classification_heads - ), "model must provide sentence classification head for --criterion=sentence_prediction" - - if not hasattr(sample, "lang_id"): - # If no language ID is given, we fall back to English - lang_id = ["en_XX"] * sample["nsentences"] - else: - lang_id = sample["lang_id"] - - logits, _ = model( - **sample["net_input"], - features_only=True, - classification_head_name=self.classification_head_name, - lang_id=lang_id, - ) - targets = model.get_targets(sample, [logits]).view(-1) - sample_size = targets.numel() - - if not self.regression_target: - lprobs = F.log_softmax(logits, dim=-1, dtype=torch.float32) - loss = F.nll_loss(lprobs, targets, reduction="sum") - else: - logits = logits.view(-1).float() - targets = targets.float() - loss = F.mse_loss(logits, targets, reduction="sum") - - logging_output = { - "loss": loss.data, - "ntokens": sample["ntokens"], - "nsentences": sample_size, - "sample_size": sample_size, - } - if not self.regression_target: - preds = logits.argmax(dim=1) - logging_output["ncorrect"] = (preds == targets).sum() - - return loss, sample_size, logging_output diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/sentence_ranking.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/sentence_ranking.py deleted file mode 100644 index d4c76341d4d87e6d0da21ac89e833ce0bda13a0c..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/sentence_ranking.py +++ /dev/null @@ -1,120 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import math - -import torch -import torch.nn.functional as F -from fairseq import metrics, utils -from fairseq.criterions import FairseqCriterion, register_criterion - - -@register_criterion("sentence_ranking") -class SentenceRankingCriterion(FairseqCriterion): - def __init__(self, task, ranking_head_name, save_predictions, num_classes): - super().__init__(task) - self.ranking_head_name = ranking_head_name - if save_predictions is not None: - self.prediction_h = open(save_predictions, "w") - else: - self.prediction_h = None - self.num_classes = num_classes - - def __del__(self): - if self.prediction_h is not None: - self.prediction_h.close() - - @staticmethod - def add_args(parser): - # fmt: off - parser.add_argument('--save-predictions', metavar='FILE', - help='file to save predictions to') - parser.add_argument('--ranking-head-name', - default='sentence_classification_head', - help='name of the ranking head to use') - # fmt: on - - def forward(self, model, sample, reduce=True): - """Compute ranking loss for the given sample. - - Returns a tuple with three elements: - 1) the loss - 2) the sample size, which is used as the denominator for the gradient - 3) logging outputs to display while training - """ - assert ( - hasattr(model, "classification_heads") - and self.ranking_head_name in model.classification_heads - ), "model must provide sentence ranking head for --criterion=sentence_ranking" - - scores = [] - for idx in range(self.num_classes): - score, _ = model( - **sample["net_input{idx}".format(idx=idx + 1)], - classification_head_name=self.ranking_head_name, - ) - scores.append(score) - - logits = torch.cat(scores, dim=1) - sample_size = logits.size(0) - - if "target" in sample: - targets = model.get_targets(sample, [logits]).view(-1) - lprobs = F.log_softmax(logits, dim=-1, dtype=torch.float32) - loss = F.nll_loss(lprobs, targets, reduction="sum") - else: - targets = None - loss = torch.tensor(0.0, requires_grad=True) - - if self.prediction_h is not None: - preds = logits.argmax(dim=1) - for i, (id, pred) in enumerate(zip(sample["id"].tolist(), preds.tolist())): - if targets is not None: - label = targets[i].item() - print("{}\t{}\t{}".format(id, pred, label), file=self.prediction_h) - else: - print("{}\t{}".format(id, pred), file=self.prediction_h) - - logging_output = { - "loss": loss.data, - "ntokens": sample["ntokens"], - "nsentences": sample_size, - "sample_size": sample_size, - } - if targets is not None: - logging_output["ncorrect"] = (logits.argmax(dim=1) == targets).sum() - - return loss, sample_size, logging_output - - @staticmethod - def reduce_metrics(logging_outputs) -> None: - """Aggregate logging outputs from data parallel training.""" - loss_sum = sum(log.get("loss", 0) for log in logging_outputs) - ntokens = sum(log.get("ntokens", 0) for log in logging_outputs) - nsentences = sum(log.get("nsentences", 0) for log in logging_outputs) - sample_size = sum(log.get("sample_size", 0) for log in logging_outputs) - - metrics.log_scalar( - "loss", loss_sum / sample_size / math.log(2), sample_size, round=3 - ) - if sample_size != ntokens: - metrics.log_scalar( - "nll_loss", loss_sum / ntokens / math.log(2), ntokens, round=3 - ) - - if len(logging_outputs) > 0 and "ncorrect" in logging_outputs[0]: - ncorrect = sum(log.get("ncorrect", 0) for log in logging_outputs) - metrics.log_scalar( - "accuracy", 100.0 * ncorrect / nsentences, nsentences, round=1 - ) - - @staticmethod - def logging_outputs_can_be_summed() -> bool: - """ - Whether the logging outputs returned by `forward` can be summed - across workers prior to calling `reduce_metrics`. Setting this - to True will improves distributed training speed. - """ - return True diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/speech_to_speech_criterion.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/speech_to_speech_criterion.py deleted file mode 100644 index bde9d156f8bb7d34f213ffe049843f47c17d3327..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/speech_to_speech_criterion.py +++ /dev/null @@ -1,310 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import math - -import torch -from fairseq import metrics, utils -from fairseq.criterions import register_criterion -from fairseq.criterions.ctc import CtcCriterion -from fairseq.criterions.label_smoothed_cross_entropy import ( - LabelSmoothedCrossEntropyCriterion, - LabelSmoothedCrossEntropyCriterionConfig, -) -from fairseq.criterions.tacotron2_loss import ( - Tacotron2Criterion, - Tacotron2CriterionConfig, -) - - -class MultitaskCriterion: - def __init__(self, multitask_tasks): - self.multitask_criterion = {} - self.multitask_loss_weight = {} - for task_name, task_obj in multitask_tasks.items(): - if task_obj.args.decoder_type == "ctc": - self.multitask_criterion[task_name] = CtcCriterion( - task_obj.args.criterion_cfg, task_obj - ) - else: - self.multitask_criterion[ - task_name - ] = LabelSmoothedCrossEntropyCriterion( - task_obj, - task_obj.args.criterion_cfg.sentence_avg, - label_smoothing=task_obj.args.criterion_cfg.label_smoothing, - ) - - def set_multitask_loss_weight(self, task_name, weight=0.0): - self.multitask_loss_weight[task_name] = weight - - def get_multitask_loss(self, model, sample, model_out): - logging_output = {} - loss = 0.0 - for task_name, task_criterion in self.multitask_criterion.items(): - layer_id = task_criterion.task.args.input_layer - if isinstance(task_criterion, CtcCriterion): - if task_criterion.task.args.input_from == "encoder": - non_padding_mask = ~model_out["encoder_padding_mask"][0] - input_lengths = non_padding_mask.long().sum(-1) - task_sample = { - "net_input": { - "src_tokens": model_out["encoder_states"][ - layer_id - ], # check batch idx - "src_lengths": input_lengths, - }, - "id": sample["id"], - } - else: - task_sample = { - "net_input": { - "src_tokens": model_out["inner_states"][layer_id], - "src_lengths": sample["target_lengths"], - }, - "id": sample["id"], - } - else: - task_sample = { - "net_input": { - "src_tokens": sample["multitask"][task_name]["net_input"][ - "prev_output_tokens" - ], - "encoder_out": { - "encoder_out": [model_out["encoder_states"][layer_id]], - "encoder_padding_mask": model_out["encoder_padding_mask"], - }, - } - } - - for key in ["target", "target_lengths", "ntokens"]: - task_sample[key] = sample["multitask"][task_name][key] - - task_loss, task_sample_size, task_logging_output = task_criterion( - model.multitask_decoders[task_name], task_sample - ) - - loss = loss + self.multitask_loss_weight[task_name] * task_loss - task_logging_output["loss_weight"] = self.multitask_loss_weight[task_name] - logging_output[task_name] = task_logging_output - return loss, logging_output - - @classmethod - def reduce_metrics(cls, logging_outputs) -> None: - for task_name in logging_outputs[0]["multitask"].keys(): - # different criterion may return different logging - # currently only reduce on loss, the most common one - # ideally the way that losses are reduced should also depend on the task type - loss_sum = sum( - log["multitask"][task_name].get("loss", 0) for log in logging_outputs - ) - sample_size = sum( - log["multitask"][task_name].get("sample_size", 0) - for log in logging_outputs - ) - - metrics.log_scalar( - f"multitask_{task_name}_loss", - loss_sum / sample_size / math.log(2), - sample_size, - round=3, - ) - - loss_weight = logging_outputs[0]["multitask"][task_name].get( - "loss_weight", 0 - ) - metrics.log_scalar( - f"multitask_{task_name}_loss_weight", - loss_weight, - weight=0, - priority=250, - ) - - -@register_criterion( - "speech_to_unit", dataclass=LabelSmoothedCrossEntropyCriterionConfig -) -class SpeechToUnitMultitaskTaskCriterion( - LabelSmoothedCrossEntropyCriterion, MultitaskCriterion -): - def __init__( - self, - task, - sentence_avg, - label_smoothing, - ignore_prefix_size=0, - report_accuracy=False, - ): - super().__init__( - task, sentence_avg, label_smoothing, ignore_prefix_size, report_accuracy - ) - MultitaskCriterion.__init__(self, task.multitask_tasks) - - def forward(self, model, sample, reduce=True): - net_output, extra = model( - src_tokens=sample["net_input"]["src_tokens"], - src_lengths=sample["net_input"]["src_lengths"], - prev_output_tokens=sample["net_input"]["prev_output_tokens"], - tgt_speaker=sample["net_input"]["tgt_speaker"], - return_all_hiddens=True, - ) - - loss, nll_loss = self.compute_loss(model, [net_output], sample, reduce=reduce) - sample_size = ( - sample["target"].size(0) if self.sentence_avg else sample["ntokens"] - ) - logging_output = { - "loss": loss.data, - "nll_loss": nll_loss.data, - "ntokens": sample["ntokens"], - "nsentences": sample["target"].size(0), - "sample_size": sample_size, - } - if self.report_accuracy: - n_correct, total = self.compute_accuracy(model, [net_output], sample) - logging_output["n_correct"] = utils.item(n_correct.data) - logging_output["total"] = utils.item(total.data) - - if len(self.multitask_criterion) == 0: - return loss, sample_size, logging_output - - # multitask - multitask_loss, multitask_log = self.get_multitask_loss(model, sample, extra) - loss += multitask_loss - logging_output["multitask"] = multitask_log - - return loss, sample_size, logging_output - - @classmethod - def reduce_metrics(cls, logging_outputs) -> None: - super().reduce_metrics(logging_outputs) - - # inference metrics - if "targ_frames" in logging_outputs[0]: - n = sum(log.get("norm_frames", 0) for log in logging_outputs) - for key, new_key in [ - ("mcd_loss", "mcd_loss"), - ("pred_frames", "pred_ratio"), - ("nins", "ins_rate"), - ("ndel", "del_rate"), - ]: - val = sum(log.get(key, 0) for log in logging_outputs) - metrics.log_scalar(new_key, val / n, n, round=3) - - if "multitask" not in logging_outputs[0]: - return - - MultitaskCriterion.reduce_metrics(logging_outputs) - - @staticmethod - def logging_outputs_can_be_summed() -> bool: - """ - Whether the logging outputs returned by `forward` can be summed - across workers prior to calling `reduce_metrics`. Setting this - to True will improves distributed training speed. - """ - return False - - -@register_criterion("speech_to_spectrogram", dataclass=Tacotron2CriterionConfig) -class SpeechToSpectrogramMultitaskTaskCriterion(Tacotron2Criterion, MultitaskCriterion): - def __init__( - self, - task, - sentence_avg, - use_guided_attention_loss, - guided_attention_loss_sigma, - bce_pos_weight, - ctc_weight, - ): - super().__init__( - task, - sentence_avg, - use_guided_attention_loss, - guided_attention_loss_sigma, - bce_pos_weight, - ctc_weight, - ) - MultitaskCriterion.__init__(self, task.multitask_tasks) - - def forward(self, model, sample, reduction="mean"): - bsz, max_len, _ = sample["target"].size() - feat_tgt = sample["target"] - feat_len = sample["target_lengths"].view(bsz, 1).expand(-1, max_len) - eos_tgt = torch.arange(max_len).to(sample["target"].device) - eos_tgt = eos_tgt.view(1, max_len).expand(bsz, -1) - eos_tgt = (eos_tgt == (feat_len - 1)).float() - - feat_out, eos_out, extra = model( - src_tokens=sample["net_input"]["src_tokens"], - src_lengths=sample["net_input"]["src_lengths"], - prev_output_tokens=sample["net_input"]["prev_output_tokens"], - tgt_speaker=sample["net_input"]["tgt_speaker"], - target_lengths=sample["target_lengths"], - return_all_hiddens=True, - ) - - l1_loss, mse_loss, eos_loss = self.compute_loss( - extra["feature_out"], - feat_out, - eos_out, - feat_tgt, - eos_tgt, - sample["target_lengths"], - reduction, - ) - attn_loss = torch.tensor(0.0).type_as(l1_loss) - if self.guided_attn is not None: - attn_loss = self.guided_attn( - extra["attn"], - sample["net_input"]["src_lengths"], - sample["target_lengths"], - reduction, - ) - loss = ( - l1_loss + mse_loss + eos_loss + attn_loss - ) # do not include ctc loss as there's no text target - - sample_size = sample["nsentences"] if self.sentence_avg else sample["ntokens"] - logging_output = { - "loss": utils.item(loss.data), - "ntokens": sample["ntokens"], - "nsentences": sample["nsentences"], - "sample_size": sample_size, - "l1_loss": utils.item(l1_loss.data), - "mse_loss": utils.item(mse_loss.data), - "eos_loss": utils.item(eos_loss.data), - "attn_loss": utils.item(attn_loss.data), - } - - if len(self.multitask_criterion) == 0: - return loss, sample_size, logging_output - - # multitask - multitask_loss, multitask_log = self.get_multitask_loss(model, sample, extra) - loss += multitask_loss - logging_output["multitask"] = multitask_log - return loss, sample_size, logging_output - - @classmethod - def reduce_metrics(cls, logging_outputs) -> None: - super().reduce_metrics(logging_outputs) - - # inference metrics - if "targ_frames" in logging_outputs[0]: - n = sum(log.get("norm_frames", 0) for log in logging_outputs) - for key, new_key in [ - ("mcd_loss", "mcd_loss"), - ("pred_frames", "pred_ratio"), - ("nins", "ins_rate"), - ("ndel", "del_rate"), - ]: - val = sum(log.get(key, 0) for log in logging_outputs) - metrics.log_scalar(new_key, val / n, n, round=3) - - if "multitask" not in logging_outputs[0]: - return - - MultitaskCriterion.reduce_metrics(logging_outputs) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/speech_ulm_criterion.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/speech_ulm_criterion.py deleted file mode 100644 index 12bb2e62b5d6625e0c82fd80c6c8ae3c383a4745..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/speech_ulm_criterion.py +++ /dev/null @@ -1,126 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from dataclasses import dataclass, field - -import torch -import torch.nn.functional as F -from fairseq import metrics -from fairseq.criterions import FairseqCriterion, register_criterion -from fairseq.dataclass import FairseqDataclass -from fairseq.tasks import FairseqTask -from omegaconf import II - - -@dataclass -class SpeechUnitLmCriterionConfig(FairseqDataclass): - sentence_avg: bool = II("optimization.sentence_avg") - loss_weights: str = field( - default="1.;0.0;0.0", - metadata={ - "help": "Weights of the losses that correspond to token, duration, and F0 streams" - }, - ) - discrete_duration: bool = II("task.discrete_duration") - discrete_f0: bool = II("task.discrete_f0") - - -def mae_loss(pred, targ, mask, reduce=True): - if pred.ndim == 3: - pred = pred.squeeze(2) - else: - assert pred.ndim == 2 - loss = (pred.float() - targ.float()).abs() * (~mask).float() - loss = loss.sum() if reduce else loss.view(-1) - return loss - - -def nll_loss(pred, targ, mask, reduce=True): - lprob = F.log_softmax(pred, dim=-1) - loss = F.nll_loss(lprob.view(-1, lprob.size(-1)), targ.view(-1), reduction="none") - loss = loss * (~mask).float().view(-1) - loss = loss.sum() if reduce else loss.view(-1) - return loss - - -@register_criterion("speech_unit_lm_criterion", dataclass=SpeechUnitLmCriterionConfig) -class SpeechUnitLmCriterion(FairseqCriterion): - def __init__(self, cfg: SpeechUnitLmCriterionConfig, task: FairseqTask): - super().__init__(task) - self.sentence_avg = cfg.sentence_avg - self.weights = torch.tensor([float(w) for w in cfg.loss_weights.split(";")]) - assert self.weights.size(0) == 3 - assert (self.weights >= 0.0).all() - - self.dur_loss_fn = nll_loss if cfg.discrete_duration else mae_loss - self.f0_loss_fn = nll_loss if cfg.discrete_f0 else mae_loss - - def forward(self, model, sample, reduce=True): - """Compute the loss for the given sample. - - Returns a tuple with three elements: - 1) the loss - 2) the sample size, which is used as the denominator for the gradient - 3) logging outputs to display while training - """ - net_output = model(**sample["net_input"]) - - token_loss = nll_loss( - net_output["token"], sample["target"], sample["mask"], reduce - ) - dur_loss = self.dur_loss_fn( - net_output["duration"], - sample["dur_target"], - sample["dur_mask"], - reduce, - ) - f0_loss = self.f0_loss_fn( - net_output["f0"], - sample["f0_target"], - sample["f0_mask"], - reduce, - ) - loss = self.weights.to(token_loss.device) * torch.stack( - [token_loss, dur_loss, f0_loss], dim=-1 - ) - loss = loss.sum() if reduce else loss.sum(-1) - - sample_size = ( - sample["target"].size(0) if self.sentence_avg else sample["ntokens"] - ) - logging_output = { - "loss": loss.detach().sum().item(), - "token_loss": token_loss.detach().sum().item(), - "dur_loss": dur_loss.detach().sum().item(), - "f0_loss": f0_loss.detach().sum().item(), - "ntokens": sample["ntokens"], - "nsentences": sample["target"].size(0), - "sample_size": sample_size, - } - return loss, sample_size, logging_output - - @staticmethod - def reduce_metrics(logging_outputs) -> None: - """Aggregate logging outputs from data parallel training.""" - loss_sum = sum(log.get("loss", 0) for log in logging_outputs) - token_loss_sum = sum(log.get("token_loss", 0) for log in logging_outputs) - dur_loss_sum = sum(log.get("dur_loss", 0) for log in logging_outputs) - f0_loss_sum = sum(log.get("f0_loss", 0) for log in logging_outputs) - - sample_size = sum(log.get("sample_size", 0) for log in logging_outputs) - - metrics.log_scalar("loss", loss_sum / sample_size, sample_size, round=3) - - metrics.log_scalar( - "token_loss", token_loss_sum / sample_size, sample_size, round=3 - ) - - metrics.log_scalar("dur_loss", dur_loss_sum / sample_size, sample_size, round=3) - - metrics.log_scalar("f0_loss", f0_loss_sum / sample_size, sample_size, round=3) - - @staticmethod - def logging_outputs_can_be_summed() -> bool: - return True diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/tacotron2_loss.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/tacotron2_loss.py deleted file mode 100644 index bfa52c7140fef755ffc99cbe75278c93e7a1a217..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/tacotron2_loss.py +++ /dev/null @@ -1,225 +0,0 @@ -# Copyright (c) 2017-present, Facebook, Inc. -# All rights reserved. -# -# This source code is licensed under the license found in the LICENSE file in -# the root directory of this source tree. An additional grant of patent rights -# can be found in the PATENTS file in the same directory. - -import logging -from dataclasses import dataclass, field -from functools import lru_cache -from typing import Any, Dict, List - -import torch -import torch.nn.functional as F -from fairseq import metrics, utils -from fairseq.criterions import FairseqCriterion, register_criterion -from fairseq.data.data_utils import lengths_to_mask -from fairseq.dataclass import FairseqDataclass -from omegaconf import II - -logger = logging.getLogger(__name__) - - -@dataclass -class Tacotron2CriterionConfig(FairseqDataclass): - bce_pos_weight: float = field( - default=1.0, - metadata={"help": "weight of positive examples for BCE loss"}, - ) - use_guided_attention_loss: bool = field( - default=False, - metadata={"help": "use guided attention loss"}, - ) - guided_attention_loss_sigma: float = field( - default=0.4, - metadata={"help": "weight of positive examples for BCE loss"}, - ) - ctc_weight: float = field(default=0.0, metadata={"help": "weight for CTC loss"}) - sentence_avg: bool = II("optimization.sentence_avg") - - -class GuidedAttentionLoss(torch.nn.Module): - """ - Efficiently Trainable Text-to-Speech System Based on Deep Convolutional - Networks with Guided Attention (https://arxiv.org/abs/1710.08969) - """ - - def __init__(self, sigma): - super().__init__() - self.sigma = sigma - - @staticmethod - @lru_cache(maxsize=8) - def _get_weight(s_len, t_len, sigma): - grid_x, grid_y = torch.meshgrid(torch.arange(t_len), torch.arange(s_len)) - grid_x = grid_x.to(s_len.device) - grid_y = grid_y.to(s_len.device) - w = (grid_y.float() / s_len - grid_x.float() / t_len) ** 2 - return 1.0 - torch.exp(-w / (2 * (sigma**2))) - - def _get_weights(self, src_lens, tgt_lens): - bsz, max_s_len, max_t_len = len(src_lens), max(src_lens), max(tgt_lens) - weights = torch.zeros((bsz, max_t_len, max_s_len)) - for i, (s_len, t_len) in enumerate(zip(src_lens, tgt_lens)): - weights[i, :t_len, :s_len] = self._get_weight(s_len, t_len, self.sigma) - return weights - - @staticmethod - def _get_masks(src_lens, tgt_lens): - in_masks = lengths_to_mask(src_lens) - out_masks = lengths_to_mask(tgt_lens) - return out_masks.unsqueeze(2) & in_masks.unsqueeze(1) - - def forward(self, attn, src_lens, tgt_lens, reduction="mean"): - weights = self._get_weights(src_lens, tgt_lens).to(attn.device) - masks = self._get_masks(src_lens, tgt_lens).to(attn.device) - loss = (weights * attn.transpose(1, 2)).masked_select(masks) - loss = torch.sum(loss) if reduction == "sum" else torch.mean(loss) - return loss - - -@register_criterion("tacotron2", dataclass=Tacotron2CriterionConfig) -class Tacotron2Criterion(FairseqCriterion): - def __init__( - self, - task, - sentence_avg, - use_guided_attention_loss, - guided_attention_loss_sigma, - bce_pos_weight, - ctc_weight, - ): - super().__init__(task) - self.sentence_avg = sentence_avg - self.bce_pos_weight = bce_pos_weight - - self.guided_attn = None - if use_guided_attention_loss: - self.guided_attn = GuidedAttentionLoss(guided_attention_loss_sigma) - self.ctc_weight = ctc_weight - - def forward(self, model, sample, reduction="mean"): - bsz, max_len, _ = sample["target"].size() - feat_tgt = sample["target"] - feat_len = sample["target_lengths"].view(bsz, 1).expand(-1, max_len) - eos_tgt = torch.arange(max_len).to(sample["target"].device) - eos_tgt = eos_tgt.view(1, max_len).expand(bsz, -1) - eos_tgt = (eos_tgt == (feat_len - 1)).float() - src_tokens = sample["net_input"]["src_tokens"] - src_lens = sample["net_input"]["src_lengths"] - tgt_lens = sample["target_lengths"] - - feat_out, eos_out, extra = model( - src_tokens=src_tokens, - src_lengths=src_lens, - prev_output_tokens=sample["net_input"]["prev_output_tokens"], - incremental_state=None, - target_lengths=tgt_lens, - speaker=sample["speaker"], - ) - - l1_loss, mse_loss, eos_loss = self.compute_loss( - extra["feature_out"], - feat_out, - eos_out, - feat_tgt, - eos_tgt, - tgt_lens, - reduction, - ) - attn_loss = torch.tensor(0.0).type_as(l1_loss) - if self.guided_attn is not None: - attn_loss = self.guided_attn(extra["attn"], src_lens, tgt_lens, reduction) - ctc_loss = torch.tensor(0.0).type_as(l1_loss) - if self.ctc_weight > 0.0: - net_output = (feat_out, eos_out, extra) - lprobs = model.get_normalized_probs(net_output, log_probs=True) - lprobs = lprobs.transpose(0, 1) # T x B x C - src_mask = lengths_to_mask(src_lens) - src_tokens_flat = src_tokens.masked_select(src_mask) - ctc_loss = ( - F.ctc_loss( - lprobs, - src_tokens_flat, - tgt_lens, - src_lens, - reduction=reduction, - zero_infinity=True, - ) - * self.ctc_weight - ) - loss = l1_loss + mse_loss + eos_loss + attn_loss + ctc_loss - - sample_size = sample["nsentences"] if self.sentence_avg else sample["ntokens"] - logging_output = { - "loss": utils.item(loss.data), - "ntokens": sample["ntokens"], - "nsentences": sample["nsentences"], - "sample_size": sample_size, - "l1_loss": utils.item(l1_loss.data), - "mse_loss": utils.item(mse_loss.data), - "eos_loss": utils.item(eos_loss.data), - "attn_loss": utils.item(attn_loss.data), - "ctc_loss": utils.item(ctc_loss.data), - } - return loss, sample_size, logging_output - - def compute_loss( - self, - feat_out, - feat_out_post, - eos_out, - feat_tgt, - eos_tgt, - tgt_lens, - reduction="mean", - ): - mask = lengths_to_mask(tgt_lens) - _eos_out = eos_out[mask].squeeze() - _eos_tgt = eos_tgt[mask] - _feat_tgt = feat_tgt[mask] - _feat_out = feat_out[mask] - _feat_out_post = feat_out_post[mask] - - l1_loss = F.l1_loss(_feat_out, _feat_tgt, reduction=reduction) + F.l1_loss( - _feat_out_post, _feat_tgt, reduction=reduction - ) - mse_loss = F.mse_loss(_feat_out, _feat_tgt, reduction=reduction) + F.mse_loss( - _feat_out_post, _feat_tgt, reduction=reduction - ) - eos_loss = F.binary_cross_entropy_with_logits( - _eos_out, - _eos_tgt, - pos_weight=torch.tensor(self.bce_pos_weight), - reduction=reduction, - ) - return l1_loss, mse_loss, eos_loss - - @classmethod - def reduce_metrics(cls, logging_outputs: List[Dict[str, Any]]) -> None: - ns = [log.get("sample_size", 0) for log in logging_outputs] - ntot = sum(ns) - ws = [n / (ntot + 1e-8) for n in ns] - for key in ["loss", "l1_loss", "mse_loss", "eos_loss", "attn_loss", "ctc_loss"]: - vals = [log.get(key, 0) for log in logging_outputs] - val = sum(val * w for val, w in zip(vals, ws)) - metrics.log_scalar(key, val, ntot, round=3) - metrics.log_scalar("sample_size", ntot, len(logging_outputs)) - - # inference metrics - if "targ_frames" not in logging_outputs[0]: - return - n = sum(log.get("targ_frames", 0) for log in logging_outputs) - for key, new_key in [ - ("mcd_loss", "mcd_loss"), - ("pred_frames", "pred_ratio"), - ("nins", "ins_rate"), - ("ndel", "del_rate"), - ]: - val = sum(log.get(key, 0) for log in logging_outputs) - metrics.log_scalar(new_key, val / n, n, round=3) - - @staticmethod - def logging_outputs_can_be_summed() -> bool: - return False diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/wav2vec_criterion.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/wav2vec_criterion.py deleted file mode 100644 index e37274d5a81a10a3b07114325b7cc2abe5cf56d9..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/criterions/wav2vec_criterion.py +++ /dev/null @@ -1,230 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import math -from dataclasses import dataclass, field -from typing import List, Optional - -import torch -import torch.nn.functional as F -from fairseq import metrics, utils -from fairseq.criterions import FairseqCriterion, register_criterion -from fairseq.dataclass import FairseqDataclass -from fairseq.logging.meters import safe_round -from fairseq.utils import is_xla_tensor - - -@dataclass -class Wav2VecCriterionConfig(FairseqDataclass): - infonce: bool = field( - default=False, - metadata={ - "help": "if set, uses cross entropy instead of binary cross entropy (i.e. InfoNCE loss)" - }, - ) - loss_weights: Optional[List[float]] = field( - default=None, - metadata={"help": "weights for additional loss terms (not first one)"}, - ) - log_keys: List[str] = field( - default_factory=lambda: [], - metadata={"help": "output keys to log"}, - ) - - -@register_criterion("wav2vec", dataclass=Wav2VecCriterionConfig) -class Wav2vecCriterion(FairseqCriterion): - def __init__(self, task, infonce=False, loss_weights=None, log_keys=None): - super().__init__(task) - self.infonce = infonce - self.loss_weights = loss_weights - self.log_keys = [] if log_keys is None else log_keys - - def forward(self, model, sample, reduce=True): - """Compute the loss for the given sample. - - Returns a tuple with three elements: - 1) the loss - 2) the sample size, which is used as the denominator for the gradient - 3) logging outputs to display while training - """ - net_output = model(**sample["net_input"]) - logits = model.get_logits(net_output).float() - target = model.get_targets(sample, net_output) - self.xla = is_xla_tensor(logits) - - # XXX: handle weights on xla. - weights = None - if hasattr(model, "get_target_weights") and not self.infonce: - weights = model.get_target_weights(target, net_output) - if torch.is_tensor(weights): - weights = weights.float() - - losses = [] - - reduction = "none" if ((not reduce) or self.xla) else "sum" - if self.infonce: - loss = F.cross_entropy(logits, target, reduction=reduction) - else: - loss = F.binary_cross_entropy_with_logits( - logits, target.float(), weights, reduction=reduction - ) - - if self.xla: - # tpu-comment: since dynamic shapes lead to recompilations on xla, - # we don't shrink tensors using mask_indices. - # Instead, we use mask indices to adjust loss. - mi = ( - sample["net_input"]["mask_indices"] - .transpose(0, 1) # logits are transposed in `model.get_logits` - .reshape(logits.size(0)) - ) - loss = (loss * mi).sum() if reduce else (loss * mi) - - if "sample_size" in sample: - sample_size = sample["sample_size"] - elif "mask_indices" in sample["net_input"]: - sample_size = sample["net_input"]["mask_indices"].sum() - else: - sample_size = target.numel() if self.infonce else target.long().sum().item() - losses.append(loss.detach().clone()) - - if self.loss_weights is not None: - assert hasattr(model, "get_extra_losses") - extra_losses = model.get_extra_losses(net_output) - if torch.is_tensor(extra_losses): - extra_losses = [extra_losses] - if len(self.loss_weights) == 1 and len(extra_losses) != 1: - self.loss_weights = [self.loss_weights[0]] * len(extra_losses) - assert len(extra_losses) == len( - self.loss_weights - ), f"{len(extra_losses)}, {len(self.loss_weights)}" - for p, coef in zip(extra_losses, self.loss_weights): - if coef != 0 and p is not None: - p = coef * p.float() * sample_size - loss += p - losses.append(p) - - logging_output = { - "loss": loss.item() if (reduce and not self.xla) else loss.detach(), - "ntokens": sample_size, - "nsentences": sample["id"].numel(), - "sample_size": sample_size, - } - - for lk in self.log_keys: - # Only store "logits" and "target" for computing MAP and MAUC - # during validation - if lk == "logits": - if not self.training: - logging_output["logits"] = logits.cpu().numpy() - elif lk == "target": - if not self.training: - # If the targets have been mixed with the predictions of - # teacher models, find the original targets - if hasattr(model, "get_original_targets"): - original_target = model.get_original_targets(sample, net_output) - else: - original_target = target - logging_output["target"] = original_target.cpu().numpy() - elif lk in net_output: - value = net_output[lk] - if not is_xla_tensor(value): - value = float(value) - logging_output[lk] = value - - if len(losses) > 1: - for i, l in enumerate(losses): - logging_output[f"loss_{i}"] = l.item() if not self.xla else l.detach() - - if self.infonce: - with torch.no_grad(): - if logits.numel() == 0: - corr = 0 - count = 0 - else: - assert logits.dim() > 1, logits.shape - max = logits.argmax(-1) == 0 - min = logits.argmin(-1) == 0 - if is_xla_tensor(logits): - max, min = max * mi, min * mi - both = max & min - corr = max.long().sum() - both.long().sum() - count = mi.sum() - else: - both = max & min - corr = max.long().sum().item() - both.long().sum().item() - count = float(max.numel()) - - logging_output["correct"] = corr - logging_output["count"] = count - - return loss, sample_size, logging_output - - @staticmethod - def reduce_metrics(logging_outputs) -> None: - """Aggregate logging outputs from data parallel training.""" - loss_sum = utils.item(sum(log.get("loss", 0) for log in logging_outputs)) - ntokens = utils.item(sum(log.get("ntokens", 0) for log in logging_outputs)) - nsentences = utils.item( - sum(log.get("nsentences", 0) for log in logging_outputs) - ) - sample_size = utils.item( - sum(log.get("sample_size", 0) for log in logging_outputs) - ) - - metrics.log_scalar( - "loss", loss_sum / (sample_size or 1) / math.log(2), sample_size, round=3 - ) - metrics.log_scalar("ntokens", ntokens) - metrics.log_scalar("nsentences", nsentences) - - correct = sum(log.get("correct", 0) for log in logging_outputs) - metrics.log_scalar("_correct", correct) - - total = sum(log.get("count", 0) for log in logging_outputs) - metrics.log_scalar("_total", total) - - if total > 0: - metrics.log_derived( - "accuracy", - lambda meters: safe_round( - meters["_correct"].sum / meters["_total"].sum, 5 - ) - if meters["_total"].sum > 0 - else float("nan"), - ) - - builtin_keys = { - "loss", - "ntokens", - "nsentences", - "sample_size", - "correct", - "count", - } - - for k in logging_outputs[0]: - if k not in builtin_keys: - val = sum(log.get(k, 0) for log in logging_outputs) - if k.startswith("loss"): - metrics.log_scalar( - k, val / (sample_size or 1) / math.log(2), sample_size, round=3 - ) - else: - metrics.log_scalar(k, val / len(logging_outputs), round=3) - - # FIXME: revert when gather based xla reduction is implemented - # @staticmethod - # def logging_outputs_can_be_summed() -> bool: - def logging_outputs_can_be_summed(self) -> bool: - """ - Whether the logging outputs returned by `forward` can be summed - across workers prior to calling `reduce_metrics`. Setting this - to True will improves distributed training speed. - """ - # XXX: Gather based reduction not implemented for xla yet. - # So we fall to sum based reduction for xla. - return self.xla diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/__init__.py deleted file mode 100644 index 8acf2ca173b44757719c4e3a7352d357b3100a0c..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/__init__.py +++ /dev/null @@ -1,130 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. -"""isort:skip_file""" - -from .dictionary import Dictionary, TruncatedDictionary - -from .fairseq_dataset import FairseqDataset, FairseqIterableDataset - -from .base_wrapper_dataset import BaseWrapperDataset - -from .add_target_dataset import AddTargetDataset -from .append_token_dataset import AppendTokenDataset -from .audio.raw_audio_dataset import BinarizedAudioDataset, FileAudioDataset -from .audio.hubert_dataset import HubertDataset -from .backtranslation_dataset import BacktranslationDataset -from .bucket_pad_length_dataset import BucketPadLengthDataset -from .colorize_dataset import ColorizeDataset -from .concat_dataset import ConcatDataset -from .concat_sentences_dataset import ConcatSentencesDataset -from .denoising_dataset import DenoisingDataset -from .id_dataset import IdDataset -from .indexed_dataset import ( - IndexedCachedDataset, - IndexedDataset, - IndexedRawTextDataset, - MMapIndexedDataset, -) -from .language_pair_dataset import LanguagePairDataset -from .list_dataset import ListDataset -from .lm_context_window_dataset import LMContextWindowDataset -from .lru_cache_dataset import LRUCacheDataset -from .mask_tokens_dataset import MaskTokensDataset -from .monolingual_dataset import MonolingualDataset -from .multi_corpus_sampled_dataset import MultiCorpusSampledDataset -from .nested_dictionary_dataset import NestedDictionaryDataset -from .noising import NoisingDataset -from .numel_dataset import NumelDataset -from .num_samples_dataset import NumSamplesDataset -from .offset_tokens_dataset import OffsetTokensDataset -from .pad_dataset import LeftPadDataset, PadDataset, RightPadDataset -from .prepend_dataset import PrependDataset -from .prepend_token_dataset import PrependTokenDataset -from .raw_label_dataset import RawLabelDataset -from .replace_dataset import ReplaceDataset -from .resampling_dataset import ResamplingDataset -from .roll_dataset import RollDataset -from .round_robin_zip_datasets import RoundRobinZipDatasets -from .sort_dataset import SortDataset -from .strip_token_dataset import StripTokenDataset -from .subsample_dataset import SubsampleDataset -from .token_block_dataset import TokenBlockDataset -from .transform_eos_dataset import TransformEosDataset -from .transform_eos_lang_pair_dataset import TransformEosLangPairDataset -from .shorten_dataset import TruncateDataset, RandomCropDataset -from .multilingual.sampled_multi_dataset import SampledMultiDataset -from .multilingual.sampled_multi_epoch_dataset import SampledMultiEpochDataset -from .fasta_dataset import FastaDataset, EncodedFastaDataset -from .transform_eos_concat_langpair_dataset import TransformEosConcatLangPairDataset - -from .iterators import ( - CountingIterator, - EpochBatchIterator, - GroupedIterator, - ShardedIterator, -) - -__all__ = [ - "AddTargetDataset", - "AppendTokenDataset", - "BacktranslationDataset", - "BaseWrapperDataset", - "BinarizedAudioDataset", - "BucketPadLengthDataset", - "ColorizeDataset", - "ConcatDataset", - "ConcatSentencesDataset", - "CountingIterator", - "DenoisingDataset", - "Dictionary", - "EncodedFastaDataset", - "EpochBatchIterator", - "FairseqDataset", - "FairseqIterableDataset", - "FastaDataset", - "FileAudioDataset", - "GroupedIterator", - "HubertDataset", - "IdDataset", - "IndexedCachedDataset", - "IndexedDataset", - "IndexedRawTextDataset", - "LanguagePairDataset", - "LeftPadDataset", - "ListDataset", - "LMContextWindowDataset", - "LRUCacheDataset", - "MaskTokensDataset", - "MMapIndexedDataset", - "MonolingualDataset", - "MultiCorpusSampledDataset", - "NestedDictionaryDataset", - "NoisingDataset", - "NumelDataset", - "NumSamplesDataset", - "OffsetTokensDataset", - "PadDataset", - "PrependDataset", - "PrependTokenDataset", - "RandomCropDataset", - "RawLabelDataset", - "ResamplingDataset", - "ReplaceDataset", - "RightPadDataset", - "RollDataset", - "RoundRobinZipDatasets", - "SampledMultiDataset", - "SampledMultiEpochDataset", - "ShardedIterator", - "SortDataset", - "StripTokenDataset", - "SubsampleDataset", - "TokenBlockDataset", - "TransformEosDataset", - "TransformEosLangPairDataset", - "TransformEosConcatLangPairDataset", - "TruncateDataset", - "TruncatedDictionary", -] diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/add_target_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/add_target_dataset.py deleted file mode 100644 index 80597674abaef3435c574e6e3026aea191af4a28..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/add_target_dataset.py +++ /dev/null @@ -1,83 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import torch -from fairseq.data.text_compressor import TextCompressionLevel, TextCompressor - -from . import BaseWrapperDataset, data_utils - - -class AddTargetDataset(BaseWrapperDataset): - def __init__( - self, - dataset, - labels, - pad, - eos, - batch_targets, - process_label=None, - label_len_fn=None, - add_to_input=False, - text_compression_level=TextCompressionLevel.none, - ): - super().__init__(dataset) - self.labels = labels - self.batch_targets = batch_targets - self.pad = pad - self.eos = eos - self.process_label = process_label - self.label_len_fn = label_len_fn - self.add_to_input = add_to_input - self.text_compressor = TextCompressor(level=text_compression_level) - - def get_label(self, index, process_fn=None): - lbl = self.labels[index] - lbl = self.text_compressor.decompress(lbl) - return lbl if process_fn is None else process_fn(lbl) - - def __getitem__(self, index): - item = self.dataset[index] - item["label"] = self.get_label(index, process_fn=self.process_label) - return item - - def size(self, index): - sz = self.dataset.size(index) - own_sz = self.label_len_fn(self.get_label(index)) - return sz, own_sz - - def collater(self, samples): - collated = self.dataset.collater(samples) - if len(collated) == 0: - return collated - indices = set(collated["id"].tolist()) - target = [s["label"] for s in samples if s["id"] in indices] - - if self.add_to_input: - eos = torch.LongTensor([self.eos]) - prev_output_tokens = [torch.cat([eos, t], axis=-1) for t in target] - target = [torch.cat([t, eos], axis=-1) for t in target] - collated["net_input"]["prev_output_tokens"] = prev_output_tokens - - if self.batch_targets: - collated["target_lengths"] = torch.LongTensor([len(t) for t in target]) - target = data_utils.collate_tokens(target, pad_idx=self.pad, left_pad=False) - collated["ntokens"] = collated["target_lengths"].sum().item() - if getattr(collated["net_input"], "prev_output_tokens", None): - collated["net_input"]["prev_output_tokens"] = data_utils.collate_tokens( - collated["net_input"]["prev_output_tokens"], - pad_idx=self.pad, - left_pad=False, - ) - else: - collated["ntokens"] = sum([len(t) for t in target]) - - collated["target"] = target - return collated - - def filter_indices_by_size(self, indices, max_sizes): - indices, ignored = data_utils._filter_by_size_dynamic( - indices, self.size, max_sizes - ) - return indices, ignored diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/append_token_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/append_token_dataset.py deleted file mode 100644 index 87695bd0f5fcb6b10247e3b743340623e6438cc1..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/append_token_dataset.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import numpy as np -import torch - -from . import BaseWrapperDataset - - -class AppendTokenDataset(BaseWrapperDataset): - def __init__(self, dataset, token=None): - super().__init__(dataset) - self.token = token - if token is not None: - self._sizes = np.array(dataset.sizes) + 1 - else: - self._sizes = dataset.sizes - - def __getitem__(self, idx): - item = self.dataset[idx] - if self.token is not None: - item = torch.cat([item, item.new([self.token])]) - return item - - @property - def sizes(self): - return self._sizes - - def num_tokens(self, index): - n = self.dataset.num_tokens(index) - if self.token is not None: - n += 1 - return n - - def size(self, index): - n = self.dataset.size(index) - if self.token is not None: - n += 1 - return n diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/audio_utils.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/audio_utils.py deleted file mode 100644 index 349b86c140ebb19f060ce08d6087cb5b0a6c3326..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/audio_utils.py +++ /dev/null @@ -1,293 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - - -import mmap -from pathlib import Path -from typing import BinaryIO, List, Optional, Tuple, Union - -import numpy as np -import torch -import torch.nn.functional as F - -SF_AUDIO_FILE_EXTENSIONS = {".wav", ".flac", ".ogg"} -FEATURE_OR_SF_AUDIO_FILE_EXTENSIONS = {".npy", ".wav", ".flac", ".ogg"} - - -def convert_waveform( - waveform: Union[np.ndarray, torch.Tensor], - sample_rate: int, - normalize_volume: bool = False, - to_mono: bool = False, - to_sample_rate: Optional[int] = None, -) -> Tuple[Union[np.ndarray, torch.Tensor], int]: - """convert a waveform: - - to a target sample rate - - from multi-channel to mono channel - - volume normalization - - Args: - waveform (numpy.ndarray or torch.Tensor): 2D original waveform - (channels x length) - sample_rate (int): original sample rate - normalize_volume (bool): perform volume normalization - to_mono (bool): convert to mono channel if having multiple channels - to_sample_rate (Optional[int]): target sample rate - Returns: - waveform (numpy.ndarray): converted 2D waveform (channels x length) - sample_rate (float): target sample rate - """ - try: - import torchaudio.sox_effects as ta_sox - except ImportError: - raise ImportError("Please install torchaudio: pip install torchaudio") - - effects = [] - if normalize_volume: - effects.append(["gain", "-n"]) - if to_sample_rate is not None and to_sample_rate != sample_rate: - effects.append(["rate", f"{to_sample_rate}"]) - if to_mono and waveform.shape[0] > 1: - effects.append(["channels", "1"]) - if len(effects) > 0: - is_np_input = isinstance(waveform, np.ndarray) - _waveform = torch.from_numpy(waveform) if is_np_input else waveform - converted, converted_sample_rate = ta_sox.apply_effects_tensor( - _waveform, sample_rate, effects - ) - if is_np_input: - converted = converted.numpy() - return converted, converted_sample_rate - return waveform, sample_rate - - -def get_waveform( - path_or_fp: Union[str, BinaryIO], - normalization: bool = True, - mono: bool = True, - frames: int = -1, - start: int = 0, - always_2d: bool = True, - output_sample_rate: Optional[int] = None, - normalize_volume: bool = False, -) -> Tuple[np.ndarray, int]: - """Get the waveform and sample rate of a 16-bit WAV/FLAC/OGG Vorbis audio. - - Args: - path_or_fp (str or BinaryIO): the path or file-like object - normalization (bool): normalize values to [-1, 1] (Default: True) - mono (bool): convert multi-channel audio to mono-channel one - frames (int): the number of frames to read. (-1 for reading all) - start (int): Where to start reading. A negative value counts from the end. - always_2d (bool): always return 2D array even for mono-channel audios - output_sample_rate (Optional[int]): output sample rate - normalize_volume (bool): normalize volume - Returns: - waveform (numpy.ndarray): 1D or 2D waveform (channels x length) - sample_rate (float): sample rate - """ - if isinstance(path_or_fp, str): - ext = Path(path_or_fp).suffix - if ext not in SF_AUDIO_FILE_EXTENSIONS: - raise ValueError(f"Unsupported audio format: {ext}") - - try: - import soundfile as sf - except ImportError: - raise ImportError("Please install soundfile: pip install soundfile") - - waveform, sample_rate = sf.read( - path_or_fp, dtype="float32", always_2d=True, frames=frames, start=start - ) - waveform = waveform.T # T x C -> C x T - waveform, sample_rate = convert_waveform( - waveform, - sample_rate, - normalize_volume=normalize_volume, - to_mono=mono, - to_sample_rate=output_sample_rate, - ) - - if not normalization: - waveform *= 2**15 # denormalized to 16-bit signed integers - if not always_2d: - waveform = waveform.squeeze(axis=0) - return waveform, sample_rate - - -def _get_kaldi_fbank( - waveform: np.ndarray, sample_rate: int, n_bins=80 -) -> Optional[np.ndarray]: - """Get mel-filter bank features via PyKaldi.""" - try: - from kaldi.feat.fbank import Fbank, FbankOptions - from kaldi.feat.mel import MelBanksOptions - from kaldi.feat.window import FrameExtractionOptions - from kaldi.matrix import Vector - - mel_opts = MelBanksOptions() - mel_opts.num_bins = n_bins - frame_opts = FrameExtractionOptions() - frame_opts.samp_freq = sample_rate - opts = FbankOptions() - opts.mel_opts = mel_opts - opts.frame_opts = frame_opts - fbank = Fbank(opts=opts) - features = fbank.compute(Vector(waveform.squeeze()), 1.0).numpy() - return features - except ImportError: - return None - - -def _get_torchaudio_fbank( - waveform: np.ndarray, sample_rate, n_bins=80 -) -> Optional[np.ndarray]: - """Get mel-filter bank features via TorchAudio.""" - try: - import torchaudio.compliance.kaldi as ta_kaldi - - waveform = torch.from_numpy(waveform) - features = ta_kaldi.fbank( - waveform, num_mel_bins=n_bins, sample_frequency=sample_rate - ) - return features.numpy() - except ImportError: - return None - - -def get_fbank(path_or_fp: Union[str, BinaryIO], n_bins=80) -> np.ndarray: - """Get mel-filter bank features via PyKaldi or TorchAudio. Prefer PyKaldi - (faster CPP implementation) to TorchAudio (Python implementation). Note that - Kaldi/TorchAudio requires 16-bit signed integers as inputs and hence the - waveform should not be normalized.""" - waveform, sample_rate = get_waveform(path_or_fp, normalization=False) - - features = _get_kaldi_fbank(waveform, sample_rate, n_bins) - if features is None: - features = _get_torchaudio_fbank(waveform, sample_rate, n_bins) - if features is None: - raise ImportError( - "Please install pyKaldi or torchaudio to enable " - "online filterbank feature extraction" - ) - - return features - - -def is_npy_data(data: bytes) -> bool: - return data[0] == 147 and data[1] == 78 - - -def is_sf_audio_data(data: bytes) -> bool: - is_wav = data[0] == 82 and data[1] == 73 and data[2] == 70 - is_flac = data[0] == 102 and data[1] == 76 and data[2] == 97 - is_ogg = data[0] == 79 and data[1] == 103 and data[2] == 103 - return is_wav or is_flac or is_ogg - - -def mmap_read(path: str, offset: int, length: int) -> bytes: - with open(path, "rb") as f: - with mmap.mmap(f.fileno(), length=0, access=mmap.ACCESS_READ) as mmap_o: - data = mmap_o[offset : offset + length] - return data - - -def read_from_stored_zip(zip_path: str, offset: int, length: int) -> bytes: - return mmap_read(zip_path, offset, length) - - -def parse_path(path: str) -> Tuple[str, List[int]]: - """Parse data path which is either a path to - 1. a .npy/.wav/.flac/.ogg file - 2. a stored ZIP file with slicing info: "[zip_path]:[offset]:[length]" - - Args: - path (str): the data path to parse - - Returns: - file_path (str): the file path - slice_ptr (list of int): empty in case 1; - byte offset and length for the slice in case 2 - """ - - if Path(path).suffix in FEATURE_OR_SF_AUDIO_FILE_EXTENSIONS: - _path, slice_ptr = path, [] - else: - _path, *slice_ptr = path.split(":") - if not Path(_path).is_file(): - raise FileNotFoundError(f"File not found: {_path}") - assert len(slice_ptr) in {0, 2}, f"Invalid path: {path}" - slice_ptr = [int(i) for i in slice_ptr] - return _path, slice_ptr - - -def get_window(window_fn: callable, n_fft: int, win_length: int) -> torch.Tensor: - padding = n_fft - win_length - assert padding >= 0 - return F.pad(window_fn(win_length), (padding // 2, padding - padding // 2)) - - -def get_fourier_basis(n_fft: int) -> torch.Tensor: - basis = np.fft.fft(np.eye(n_fft)) - basis = np.vstack( - [np.real(basis[: n_fft // 2 + 1, :]), np.imag(basis[: n_fft // 2 + 1, :])] - ) - return torch.from_numpy(basis).float() - - -def get_mel_filters( - sample_rate: int, n_fft: int, n_mels: int, f_min: float, f_max: float -) -> torch.Tensor: - try: - import librosa - except ImportError: - raise ImportError("Please install librosa: pip install librosa") - basis = librosa.filters.mel(sample_rate, n_fft, n_mels, f_min, f_max) - return torch.from_numpy(basis).float() - - -class TTSSpectrogram(torch.nn.Module): - def __init__( - self, - n_fft: int, - win_length: int, - hop_length: int, - window_fn: callable = torch.hann_window, - return_phase: bool = False, - ) -> None: - super(TTSSpectrogram, self).__init__() - self.n_fft = n_fft - self.hop_length = hop_length - self.return_phase = return_phase - - basis = get_fourier_basis(n_fft).unsqueeze(1) - basis *= get_window(window_fn, n_fft, win_length) - self.register_buffer("basis", basis) - - def forward( - self, waveform: torch.Tensor - ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: - padding = (self.n_fft // 2, self.n_fft // 2) - x = F.pad(waveform.unsqueeze(1), padding, mode="reflect") - x = F.conv1d(x, self.basis, stride=self.hop_length) - real_part = x[:, : self.n_fft // 2 + 1, :] - imag_part = x[:, self.n_fft // 2 + 1 :, :] - magnitude = torch.sqrt(real_part**2 + imag_part**2) - if self.return_phase: - phase = torch.atan2(imag_part, real_part) - return magnitude, phase - return magnitude - - -class TTSMelScale(torch.nn.Module): - def __init__( - self, n_mels: int, sample_rate: int, f_min: float, f_max: float, n_stft: int - ) -> None: - super(TTSMelScale, self).__init__() - basis = get_mel_filters(sample_rate, (n_stft - 1) * 2, n_mels, f_min, f_max) - self.register_buffer("basis", basis) - - def forward(self, specgram: torch.Tensor) -> torch.Tensor: - return torch.matmul(self.basis, specgram) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/data_cfg.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/data_cfg.py deleted file mode 100644 index fba36dfcf07c35ac21fd77f0b58837fd002a3e3a..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/data_cfg.py +++ /dev/null @@ -1,299 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. -from argparse import Namespace -from pathlib import Path -from typing import Dict, Optional - -from fairseq.data import Dictionary - - -def get_config_from_yaml(yaml_path: Path): - try: - import yaml - except ImportError: - print("Please install PyYAML: pip install PyYAML") - config = {} - if yaml_path.is_file(): - try: - with open(yaml_path) as f: - config = yaml.load(f, Loader=yaml.FullLoader) - except Exception as e: - raise Exception(f"Failed to load config from {yaml_path.as_posix()}: {e}") - else: - raise FileNotFoundError(f"{yaml_path.as_posix()} not found") - - return config - - -class S2TDataConfig(object): - """Wrapper class for data config YAML""" - - def __init__(self, yaml_path: Path): - self.config = get_config_from_yaml(yaml_path) - self.root = yaml_path.parent - - def _auto_convert_to_abs_path(self, x): - if isinstance(x, str): - if not Path(x).exists() and (self.root / x).exists(): - return (self.root / x).as_posix() - elif isinstance(x, dict): - return {k: self._auto_convert_to_abs_path(v) for k, v in x.items()} - return x - - @property - def vocab_filename(self): - """fairseq vocabulary file under data root""" - return self.config.get("vocab_filename", "dict.txt") - - @property - def speaker_set_filename(self): - """speaker set file under data root""" - return self.config.get("speaker_set_filename", None) - - @property - def shuffle(self) -> bool: - """Shuffle dataset samples before batching""" - return self.config.get("shuffle", False) - - @property - def pre_tokenizer(self) -> Dict: - """Pre-tokenizer to apply before subword tokenization. Returning - a dictionary with `tokenizer` providing the tokenizer name and - the other items providing the tokenizer-specific arguments. - Tokenizers are defined in `fairseq.data.encoders.*`""" - tokenizer = self.config.get("pre_tokenizer", {"tokenizer": None}) - return self._auto_convert_to_abs_path(tokenizer) - - @property - def bpe_tokenizer(self) -> Dict: - """Subword tokenizer to apply after pre-tokenization. Returning - a dictionary with `bpe` providing the tokenizer name and - the other items providing the tokenizer-specific arguments. - Tokenizers are defined in `fairseq.data.encoders.*`""" - tokenizer = self.config.get("bpe_tokenizer", {"bpe": None}) - return self._auto_convert_to_abs_path(tokenizer) - - @property - def prepend_tgt_lang_tag(self) -> bool: - """Prepend target lang ID token as the target BOS (e.g. for to-many - multilingual setting). During inference, this requires `--prefix-size 1` - to force BOS to be lang ID token.""" - return self.config.get("prepend_tgt_lang_tag", False) - - @property - def prepend_bos_and_append_tgt_lang_tag(self) -> bool: - """Prepend BOS and append target lang ID token to the target (e.g. mBART with language token pretraining).""" - return self.config.get("prepend_bos_and_append_tgt_lang_tag", False) - - @property - def input_feat_per_channel(self): - """The dimension of input features (per audio channel)""" - return self.config.get("input_feat_per_channel", 80) - - @property - def input_channels(self): - """The number of channels in the input audio""" - return self.config.get("input_channels", 1) - - @property - def sample_rate(self): - return self.config.get("sample_rate", 16_000) - - @property - def sampling_alpha(self): - """Hyper-parameter alpha = 1/T for temperature-based resampling. - (alpha = 1 for no resampling)""" - return self.config.get("sampling_alpha", 1.0) - - @property - def use_audio_input(self): - """Needed by the dataset loader to see if the model requires - raw audio as inputs.""" - return self.config.get("use_audio_input", False) - - def standardize_audio(self) -> bool: - return self.use_audio_input and self.config.get("standardize_audio", False) - - @property - def use_sample_rate(self): - """Needed by the dataset loader to see if the model requires - raw audio with specific sample rate as inputs.""" - return self.config.get("use_sample_rate", 16000) - - @property - def audio_root(self): - """Audio paths in the manifest TSV can be relative and this provides - the root path. Set this to empty string when using absolute paths.""" - return self.config.get("audio_root", "") - - def get_feature_transforms(self, split, is_train): - """Split-specific feature transforms. Allowing train set - wildcard `_train`, evaluation set wildcard `_eval` and general - wildcard `*` for matching.""" - from copy import deepcopy - - cfg = deepcopy(self.config) - _cur = cfg.get("transforms", {}) - cur = _cur.get(split) - cur = _cur.get("_train") if cur is None and is_train else cur - cur = _cur.get("_eval") if cur is None and not is_train else cur - cur = _cur.get("*") if cur is None else cur - cfg["transforms"] = cur - return cfg - - @property - def global_cmvn_stats_npz(self) -> Optional[str]: - path = self.config.get("global_cmvn", {}).get("stats_npz_path", None) - return self._auto_convert_to_abs_path(path) - - @property - def vocoder(self) -> Dict[str, str]: - vocoder = self.config.get("vocoder", {"type": "griffin_lim"}) - return self._auto_convert_to_abs_path(vocoder) - - @property - def hub(self) -> Dict[str, str]: - return self.config.get("hub", {}) - - -class S2SDataConfig(S2TDataConfig): - """Wrapper class for data config YAML""" - - @property - def vocab_filename(self): - """fairseq vocabulary file under data root""" - return self.config.get("vocab_filename", None) - - @property - def pre_tokenizer(self) -> Dict: - return None - - @property - def bpe_tokenizer(self) -> Dict: - return None - - @property - def input_transformed_channels(self): - """The number of channels in the audio after feature transforms""" - # TODO: move this into individual transforms - _cur = self.config.get("transforms", {}) - cur = _cur.get("_train", []) - - _channels = self.input_channels - if "delta_deltas" in cur: - _channels *= 3 - - return _channels - - @property - def output_sample_rate(self): - """The audio sample rate of output target speech""" - return self.config.get("output_sample_rate", 22050) - - @property - def target_speaker_embed(self): - """Target speaker embedding file (one line per target audio sample)""" - return self.config.get("target_speaker_embed", None) - - @property - def prepend_tgt_lang_tag_as_bos(self) -> bool: - """Prepend target lang ID token as the target BOS.""" - return self.config.get("prepend_tgt_lang_tag_as_bos", False) - - -class MultitaskConfig(object): - """Wrapper class for data config YAML""" - - def __init__(self, yaml_path: Path): - config = get_config_from_yaml(yaml_path) - self.config = {} - for k, v in config.items(): - self.config[k] = SingleTaskConfig(k, v) - - def get_all_tasks(self): - return self.config - - def get_single_task(self, name): - assert name in self.config, f"multitask '{name}' does not exist!" - return self.config[name] - - -class SingleTaskConfig(object): - def __init__(self, name, config): - self.task_name = name - self.config = config - dict_path = config.get("dict", "") - self.tgt_dict = Dictionary.load(dict_path) if Path(dict_path).exists() else None - - @property - def data(self): - return self.config.get("data", "") - - @property - def decoder_type(self): - return self.config.get("decoder_type", "transformer") - - @property - def decoder_args(self): - """Decoder arch related args""" - args = self.config.get("decoder_args", {}) - return Namespace(**args) - - @property - def criterion_cfg(self): - """cfg for the multitask criterion""" - if self.decoder_type == "ctc": - from fairseq.criterions.ctc import CtcCriterionConfig - - cfg = CtcCriterionConfig - cfg.zero_infinity = self.config.get("zero_infinity", True) - else: - from fairseq.criterions.label_smoothed_cross_entropy import ( - LabelSmoothedCrossEntropyCriterionConfig, - ) - - cfg = LabelSmoothedCrossEntropyCriterionConfig - cfg.label_smoothing = self.config.get("label_smoothing", 0.2) - return cfg - - @property - def input_from(self): - """Condition on encoder/decoder of the main model""" - return "decoder" if "decoder_layer" in self.config else "encoder" - - @property - def input_layer(self): - if self.input_from == "decoder": - return self.config["decoder_layer"] - 1 - else: - # default using the output from the last encoder layer (-1) - return self.config.get("encoder_layer", 0) - 1 - - @property - def loss_weight_schedule(self): - return ( - "decay" - if "loss_weight_max" in self.config - and "loss_weight_decay_steps" in self.config - else "fixed" - ) - - def get_loss_weight(self, num_updates): - if self.loss_weight_schedule == "fixed": - weight = self.config.get("loss_weight", 1.0) - else: # "decay" - assert ( - self.config.get("loss_weight_decay_steps", 0) > 0 - ), "loss_weight_decay_steps must be greater than 0 for a decay schedule" - loss_weight_min = self.config.get("loss_weight_min", 0.0001) - loss_weight_decay_stepsize = ( - self.config["loss_weight_max"] - loss_weight_min - ) / self.config["loss_weight_decay_steps"] - weight = max( - self.config["loss_weight_max"] - - loss_weight_decay_stepsize * num_updates, - loss_weight_min, - ) - return weight diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/feature_transforms/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/feature_transforms/__init__.py deleted file mode 100644 index 359fa069716cba0dd615ce0959368b20828c31f7..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/feature_transforms/__init__.py +++ /dev/null @@ -1,82 +0,0 @@ -import importlib -import os -from abc import ABC, abstractmethod -from typing import Dict, Optional - - -class AudioFeatureTransform(ABC): - @classmethod - @abstractmethod - def from_config_dict(cls, config: Optional[Dict] = None): - pass - - -AUDIO_FEATURE_TRANSFORM_REGISTRY = {} -AUDIO_FEATURE_TRANSFORM_CLASS_NAMES = set() - - -def register_audio_feature_transform(name): - def register_audio_feature_transform_cls(cls): - if name in AUDIO_FEATURE_TRANSFORM_REGISTRY: - raise ValueError(f"Cannot register duplicate transform ({name})") - if not issubclass(cls, AudioFeatureTransform): - raise ValueError( - f"Transform ({name}: {cls.__name__}) must extend " - "AudioFeatureTransform" - ) - if cls.__name__ in AUDIO_FEATURE_TRANSFORM_CLASS_NAMES: - raise ValueError( - f"Cannot register audio feature transform with duplicate " - f"class name ({cls.__name__})" - ) - AUDIO_FEATURE_TRANSFORM_REGISTRY[name] = cls - AUDIO_FEATURE_TRANSFORM_CLASS_NAMES.add(cls.__name__) - return cls - - return register_audio_feature_transform_cls - - -def get_audio_feature_transform(name): - return AUDIO_FEATURE_TRANSFORM_REGISTRY[name] - - -transforms_dir = os.path.dirname(__file__) -for file in os.listdir(transforms_dir): - path = os.path.join(transforms_dir, file) - if ( - not file.startswith("_") - and not file.startswith(".") - and (file.endswith(".py") or os.path.isdir(path)) - ): - name = file[: file.find(".py")] if file.endswith(".py") else file - importlib.import_module("fairseq.data.audio.feature_transforms." + name) - - -class CompositeAudioFeatureTransform(AudioFeatureTransform): - @classmethod - def from_config_dict(cls, config=None): - _config = {} if config is None else config - _transforms = _config.get("transforms") - if _transforms is None: - return None - transforms = [ - get_audio_feature_transform(_t).from_config_dict(_config.get(_t)) - for _t in _transforms - ] - return CompositeAudioFeatureTransform(transforms) - - def __init__(self, transforms): - self.transforms = [t for t in transforms if t is not None] - - def __call__(self, x): - for t in self.transforms: - x = t(x) - return x - - def __repr__(self): - format_string = ( - [self.__class__.__name__ + "("] - + [f" {t.__repr__()}" for t in self.transforms] - + [")"] - ) - return "\n".join(format_string) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/feature_transforms/delta_deltas.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/feature_transforms/delta_deltas.py deleted file mode 100644 index 49d090b11e5b31562e0aedc9b4e2b8d0d510eeda..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/feature_transforms/delta_deltas.py +++ /dev/null @@ -1,37 +0,0 @@ -import numpy as np -import torch -from fairseq.data.audio.feature_transforms import ( - AudioFeatureTransform, - register_audio_feature_transform, -) - - -@register_audio_feature_transform("delta_deltas") -class DeltaDeltas(AudioFeatureTransform): - """Expand delta-deltas features from spectrum.""" - - @classmethod - def from_config_dict(cls, config=None): - _config = {} if config is None else config - return DeltaDeltas(_config.get("win_length", 5)) - - def __init__(self, win_length=5): - self.win_length = win_length - - def __repr__(self): - return self.__class__.__name__ - - def __call__(self, spectrogram): - from torchaudio.functional import compute_deltas - - assert len(spectrogram.shape) == 2, "spectrogram must be a 2-D tensor." - # spectrogram is T x F, while compute_deltas takes (…, F, T) - spectrogram = torch.from_numpy(spectrogram).transpose(0, 1) - delta = compute_deltas(spectrogram) - delta_delta = compute_deltas(delta) - - out_feat = np.concatenate( - [spectrogram, delta.numpy(), delta_delta.numpy()], axis=0 - ) - out_feat = np.transpose(out_feat) - return out_feat diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/feature_transforms/global_cmvn.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/feature_transforms/global_cmvn.py deleted file mode 100644 index e457ff176fee3b996da11f47e7dc61b81c445ba3..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/feature_transforms/global_cmvn.py +++ /dev/null @@ -1,29 +0,0 @@ -import numpy as np -from fairseq.data.audio.feature_transforms import ( - AudioFeatureTransform, - register_audio_feature_transform, -) - - -@register_audio_feature_transform("global_cmvn") -class GlobalCMVN(AudioFeatureTransform): - """Global CMVN (cepstral mean and variance normalization). The global mean - and variance need to be pre-computed and stored in NumPy format (.npz).""" - - @classmethod - def from_config_dict(cls, config=None): - _config = {} if config is None else config - return GlobalCMVN(_config.get("stats_npz_path")) - - def __init__(self, stats_npz_path): - self.stats_npz_path = stats_npz_path - stats = np.load(stats_npz_path) - self.mean, self.std = stats["mean"], stats["std"] - - def __repr__(self): - return self.__class__.__name__ + f'(stats_npz_path="{self.stats_npz_path}")' - - def __call__(self, x): - x = np.subtract(x, self.mean) - x = np.divide(x, self.std) - return x diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/feature_transforms/specaugment.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/feature_transforms/specaugment.py deleted file mode 100644 index ce5802b41a903ea8f3e3e8a169d5048b4e908f99..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/feature_transforms/specaugment.py +++ /dev/null @@ -1,131 +0,0 @@ -import math -import numbers -from typing import Optional - -import numpy as np -from fairseq.data.audio.feature_transforms import ( - AudioFeatureTransform, - register_audio_feature_transform, -) - - -@register_audio_feature_transform("specaugment") -class SpecAugmentTransform(AudioFeatureTransform): - """SpecAugment (https://arxiv.org/abs/1904.08779)""" - - @classmethod - def from_config_dict(cls, config=None): - _config = {} if config is None else config - return SpecAugmentTransform( - _config.get("time_warp_W", 0), - _config.get("freq_mask_N", 0), - _config.get("freq_mask_F", 0), - _config.get("time_mask_N", 0), - _config.get("time_mask_T", 0), - _config.get("time_mask_p", 0.0), - _config.get("mask_value", None), - ) - - def __init__( - self, - time_warp_w: int = 0, - freq_mask_n: int = 0, - freq_mask_f: int = 0, - time_mask_n: int = 0, - time_mask_t: int = 0, - time_mask_p: float = 0.0, - mask_value: Optional[float] = 0.0, - ): - # Sanity checks - assert mask_value is None or isinstance( - mask_value, numbers.Number - ), f"mask_value (type: {type(mask_value)}) must be None or a number" - if freq_mask_n > 0: - assert freq_mask_f > 0, ( - f"freq_mask_F ({freq_mask_f}) " - f"must be larger than 0 when doing freq masking." - ) - if time_mask_n > 0: - assert time_mask_t > 0, ( - f"time_mask_T ({time_mask_t}) must be larger than 0 when " - f"doing time masking." - ) - - self.time_warp_w = time_warp_w - self.freq_mask_n = freq_mask_n - self.freq_mask_f = freq_mask_f - self.time_mask_n = time_mask_n - self.time_mask_t = time_mask_t - self.time_mask_p = time_mask_p - self.mask_value = mask_value - - def __repr__(self): - return ( - self.__class__.__name__ - + "(" - + ", ".join( - [ - f"time_warp_w={self.time_warp_w}", - f"freq_mask_n={self.freq_mask_n}", - f"freq_mask_f={self.freq_mask_f}", - f"time_mask_n={self.time_mask_n}", - f"time_mask_t={self.time_mask_t}", - f"time_mask_p={self.time_mask_p}", - ] - ) - + ")" - ) - - def __call__(self, spectrogram): - assert len(spectrogram.shape) == 2, "spectrogram must be a 2-D tensor." - - distorted = spectrogram.copy() # make a copy of input spectrogram. - num_frames = spectrogram.shape[0] # or 'tau' in the paper. - num_freqs = spectrogram.shape[1] # or 'miu' in the paper. - mask_value = self.mask_value - - if mask_value is None: # if no value was specified, use local mean. - mask_value = spectrogram.mean() - - if num_frames == 0: - return spectrogram - - if num_freqs < self.freq_mask_f: - return spectrogram - - if self.time_warp_w > 0: - if 2 * self.time_warp_w < num_frames: - import cv2 - - w0 = np.random.randint(self.time_warp_w, num_frames - self.time_warp_w) - w = np.random.randint(-self.time_warp_w + 1, self.time_warp_w) - upper, lower = distorted[:w0, :], distorted[w0:, :] - upper = cv2.resize( - upper, dsize=(num_freqs, w0 + w), interpolation=cv2.INTER_LINEAR - ) - lower = cv2.resize( - lower, - dsize=(num_freqs, num_frames - w0 - w), - interpolation=cv2.INTER_LINEAR, - ) - distorted = np.concatenate((upper, lower), axis=0) - - for _i in range(self.freq_mask_n): - f = np.random.randint(0, self.freq_mask_f) - f0 = np.random.randint(0, num_freqs - f) - if f != 0: - distorted[:, f0 : f0 + f] = mask_value - - max_time_mask_t = min( - self.time_mask_t, math.floor(num_frames * self.time_mask_p) - ) - if max_time_mask_t < 1: - return distorted - - for _i in range(self.time_mask_n): - t = np.random.randint(0, max_time_mask_t) - t0 = np.random.randint(0, num_frames - t) - if t != 0: - distorted[t0 : t0 + t, :] = mask_value - - return distorted diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/feature_transforms/utterance_cmvn.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/feature_transforms/utterance_cmvn.py deleted file mode 100644 index 4b32225011b52230b7816769d158fe5fba329bbf..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/feature_transforms/utterance_cmvn.py +++ /dev/null @@ -1,40 +0,0 @@ -import numpy as np -from fairseq.data.audio.feature_transforms import ( - AudioFeatureTransform, - register_audio_feature_transform, -) - - -@register_audio_feature_transform("utterance_cmvn") -class UtteranceCMVN(AudioFeatureTransform): - """Utterance-level CMVN (cepstral mean and variance normalization)""" - - @classmethod - def from_config_dict(cls, config=None): - _config = {} if config is None else config - return UtteranceCMVN( - _config.get("norm_means", True), - _config.get("norm_vars", True), - ) - - def __init__(self, norm_means=True, norm_vars=True): - self.norm_means, self.norm_vars = norm_means, norm_vars - - def __repr__(self): - return ( - self.__class__.__name__ - + f"(norm_means={self.norm_means}, norm_vars={self.norm_vars})" - ) - - def __call__(self, x): - mean = x.mean(axis=0) - square_sums = (x**2).sum(axis=0) - - if self.norm_means: - x = np.subtract(x, mean) - if self.norm_vars: - var = square_sums / x.shape[0] - mean**2 - std = np.sqrt(np.maximum(var, 1e-10)) - x = np.divide(x, std) - - return x diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/frm_text_to_speech_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/frm_text_to_speech_dataset.py deleted file mode 100644 index b54654d49250605255b375e094457062e53fa55c..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/frm_text_to_speech_dataset.py +++ /dev/null @@ -1,205 +0,0 @@ -# Copyright (c) 2017-present, Facebook, Inc. -# All rights reserved. -# -# This source code is licensed under the license found in the LICENSE file in -# the root directory of this source tree. An additional grant of patent rights -# can be found in the PATENTS file in the same directory.abs - -import csv -import logging -import os.path as op -from typing import List, Optional - -import numpy as np -import torch -from fairseq.data import Dictionary -from fairseq.data.audio.speech_to_text_dataset import S2TDataConfig -from fairseq.data.audio.text_to_speech_dataset import ( - TextToSpeechDataset, - TextToSpeechDatasetCreator, -) - -logger = logging.getLogger(__name__) - - -class FrmTextToSpeechDataset(TextToSpeechDataset): - def __init__( - self, - split: str, - is_train_split: bool, - data_cfg: S2TDataConfig, - audio_paths: List[str], - n_frames: List[int], - src_texts: Optional[List[str]] = None, - tgt_texts: Optional[List[str]] = None, - speakers: Optional[List[str]] = None, - src_langs: Optional[List[str]] = None, - tgt_langs: Optional[List[str]] = None, - ids: Optional[List[str]] = None, - tgt_dict: Optional[Dictionary] = None, - pre_tokenizer=None, - bpe_tokenizer=None, - n_frames_per_step=1, - speaker_to_id=None, - do_chunk=False, - chunk_bound=-1, - chunk_init=50, - chunk_incr=5, - add_eos=True, - dedup=True, - ref_fpu=-1, - ): - # It assumes texts are encoded at a fixed frame-rate - super().__init__( - split=split, - is_train_split=is_train_split, - data_cfg=data_cfg, - audio_paths=audio_paths, - n_frames=n_frames, - src_texts=src_texts, - tgt_texts=tgt_texts, - speakers=speakers, - src_langs=src_langs, - tgt_langs=tgt_langs, - ids=ids, - tgt_dict=tgt_dict, - pre_tokenizer=pre_tokenizer, - bpe_tokenizer=bpe_tokenizer, - n_frames_per_step=n_frames_per_step, - speaker_to_id=speaker_to_id, - ) - - self.do_chunk = do_chunk - self.chunk_bound = chunk_bound - self.chunk_init = chunk_init - self.chunk_incr = chunk_incr - self.add_eos = add_eos - self.dedup = dedup - self.ref_fpu = ref_fpu - - self.chunk_size = -1 - - if do_chunk: - assert self.chunk_incr >= 0 - assert self.pre_tokenizer is None - - def __getitem__(self, index): - index, source, target, speaker_id, _, _, _ = super().__getitem__(index) - if target[-1].item() == self.tgt_dict.eos_index: - target = target[:-1] - - fpu = source.size(0) / target.size(0) # frame-per-unit - fps = self.n_frames_per_step - assert ( - self.ref_fpu == -1 or abs((fpu * fps - self.ref_fpu) / self.ref_fpu) < 0.1 - ), f"{fpu*fps} != {self.ref_fpu}" - - # only chunk training split - if self.is_train_split and self.do_chunk and self.chunk_size > 0: - lang = target[: int(self.data_cfg.prepend_tgt_lang_tag)] - text = target[int(self.data_cfg.prepend_tgt_lang_tag) :] - size = len(text) - chunk_size = min(self.chunk_size, size) - chunk_start = np.random.randint(size - chunk_size + 1) - text = text[chunk_start : chunk_start + chunk_size] - target = torch.cat((lang, text), 0) - - f_size = int(np.floor(chunk_size * fpu)) - f_start = int(np.floor(chunk_start * fpu)) - assert f_size > 0 - source = source[f_start : f_start + f_size, :] - - if self.dedup: - target = torch.unique_consecutive(target) - - if self.add_eos: - eos_idx = self.tgt_dict.eos_index - target = torch.cat((target, torch.LongTensor([eos_idx])), 0) - - return index, source, target, speaker_id - - def set_epoch(self, epoch): - if self.is_train_split and self.do_chunk: - old = self.chunk_size - self.chunk_size = self.chunk_init + epoch * self.chunk_incr - if self.chunk_bound > 0: - self.chunk_size = min(self.chunk_size, self.chunk_bound) - logger.info( - ( - f"{self.split}: setting chunk size " - f"from {old} to {self.chunk_size}" - ) - ) - - -class FrmTextToSpeechDatasetCreator(TextToSpeechDatasetCreator): - # inherit for key names - @classmethod - def from_tsv( - cls, - root: str, - data_cfg: S2TDataConfig, - split: str, - tgt_dict, - pre_tokenizer, - bpe_tokenizer, - is_train_split: bool, - n_frames_per_step: int, - speaker_to_id, - do_chunk: bool = False, - chunk_bound: int = -1, - chunk_init: int = 50, - chunk_incr: int = 5, - add_eos: bool = True, - dedup: bool = True, - ref_fpu: float = -1, - ) -> FrmTextToSpeechDataset: - tsv_path = op.join(root, f"{split}.tsv") - if not op.isfile(tsv_path): - raise FileNotFoundError(f"Dataset not found: {tsv_path}") - with open(tsv_path) as f: - reader = csv.DictReader( - f, - delimiter="\t", - quotechar=None, - doublequote=False, - lineterminator="\n", - quoting=csv.QUOTE_NONE, - ) - s = [dict(e) for e in reader] - assert len(s) > 0 - - ids = [ss[cls.KEY_ID] for ss in s] - audio_paths = [op.join(data_cfg.audio_root, ss[cls.KEY_AUDIO]) for ss in s] - n_frames = [int(ss[cls.KEY_N_FRAMES]) for ss in s] - tgt_texts = [ss[cls.KEY_TGT_TEXT] for ss in s] - src_texts = [ss.get(cls.KEY_SRC_TEXT, cls.DEFAULT_SRC_TEXT) for ss in s] - speakers = [ss.get(cls.KEY_SPEAKER, cls.DEFAULT_SPEAKER) for ss in s] - src_langs = [ss.get(cls.KEY_SRC_LANG, cls.DEFAULT_LANG) for ss in s] - tgt_langs = [ss.get(cls.KEY_TGT_LANG, cls.DEFAULT_LANG) for ss in s] - - return FrmTextToSpeechDataset( - split=split, - is_train_split=is_train_split, - data_cfg=data_cfg, - audio_paths=audio_paths, - n_frames=n_frames, - src_texts=src_texts, - tgt_texts=tgt_texts, - speakers=speakers, - src_langs=src_langs, - tgt_langs=tgt_langs, - ids=ids, - tgt_dict=tgt_dict, - pre_tokenizer=pre_tokenizer, - bpe_tokenizer=bpe_tokenizer, - n_frames_per_step=n_frames_per_step, - speaker_to_id=speaker_to_id, - do_chunk=do_chunk, - chunk_bound=chunk_bound, - chunk_init=chunk_init, - chunk_incr=chunk_incr, - add_eos=add_eos, - dedup=dedup, - ref_fpu=ref_fpu, - ) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/hubert_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/hubert_dataset.py deleted file mode 100644 index 7e497acfaeab9448bb027de342ec1f922a6a9930..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/hubert_dataset.py +++ /dev/null @@ -1,343 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import itertools -import logging -import os -import sys -from typing import Any, List, Optional, Union - -import numpy as np -import torch -import torch.nn.functional as F -from fairseq.data import data_utils -from fairseq.data.fairseq_dataset import FairseqDataset - -logger = logging.getLogger(__name__) - - -def load_audio(manifest_path, max_keep, min_keep): - n_long, n_short = 0, 0 - names, inds, sizes = [], [], [] - with open(manifest_path) as f: - root = f.readline().strip() - for ind, line in enumerate(f): - items = line.strip().split("\t") - assert len(items) == 2, line - sz = int(items[1]) - if min_keep is not None and sz < min_keep: - n_short += 1 - elif max_keep is not None and sz > max_keep: - n_long += 1 - else: - names.append(items[0]) - inds.append(ind) - sizes.append(sz) - tot = ind + 1 - logger.info( - ( - f"max_keep={max_keep}, min_keep={min_keep}, " - f"loaded {len(names)}, skipped {n_short} short and {n_long} long, " - f"longest-loaded={max(sizes)}, shortest-loaded={min(sizes)}" - ) - ) - return root, names, inds, tot, sizes - - -def load_label(label_path, inds, tot): - with open(label_path) as f: - labels = [line.rstrip() for line in f] - assert ( - len(labels) == tot - ), f"number of labels does not match ({len(labels)} != {tot})" - labels = [labels[i] for i in inds] - return labels - - -def load_label_offset(label_path, inds, tot): - with open(label_path) as f: - code_lengths = [len(line.encode("utf-8")) for line in f] - assert ( - len(code_lengths) == tot - ), f"number of labels does not match ({len(code_lengths)} != {tot})" - offsets = list(itertools.accumulate([0] + code_lengths)) - offsets = [(offsets[i], offsets[i + 1]) for i in inds] - return offsets - - -def verify_label_lengths( - audio_sizes, - audio_rate, - label_path, - label_rate, - inds, - tot, - tol=0.1, # tolerance in seconds -): - if label_rate < 0: - logger.info(f"{label_path} is sequence label. skipped") - return - - with open(label_path) as f: - lengths = [len(line.rstrip().split()) for line in f] - assert len(lengths) == tot - lengths = [lengths[i] for i in inds] - num_invalid = 0 - for i, ind in enumerate(inds): - dur_from_audio = audio_sizes[i] / audio_rate - dur_from_label = lengths[i] / label_rate - if abs(dur_from_audio - dur_from_label) > tol: - logger.warning( - ( - f"audio and label duration differ too much " - f"(|{dur_from_audio} - {dur_from_label}| > {tol}) " - f"in line {ind+1} of {label_path}. Check if `label_rate` " - f"is correctly set (currently {label_rate}). " - f"num. of samples = {audio_sizes[i]}; " - f"label length = {lengths[i]}" - ) - ) - num_invalid += 1 - if num_invalid > 0: - logger.warning( - f"total {num_invalid} (audio, label) pairs with mismatched lengths" - ) - - -class HubertDataset(FairseqDataset): - def __init__( - self, - manifest_path: str, - sample_rate: float, - label_paths: List[str], - label_rates: Union[List[float], float], # -1 for sequence labels - pad_list: List[str], - eos_list: List[str], - label_processors: Optional[List[Any]] = None, - max_keep_sample_size: Optional[int] = None, - min_keep_sample_size: Optional[int] = None, - max_sample_size: Optional[int] = None, - shuffle: bool = True, - pad_audio: bool = False, - normalize: bool = False, - store_labels: bool = True, - random_crop: bool = False, - single_target: bool = False, - ): - self.audio_root, self.audio_names, inds, tot, self.sizes = load_audio( - manifest_path, max_keep_sample_size, min_keep_sample_size - ) - self.sample_rate = sample_rate - self.shuffle = shuffle - self.random_crop = random_crop - - self.num_labels = len(label_paths) - self.pad_list = pad_list - self.eos_list = eos_list - self.label_processors = label_processors - self.single_target = single_target - self.label_rates = ( - [label_rates for _ in range(len(label_paths))] - if isinstance(label_rates, float) - else label_rates - ) - self.store_labels = store_labels - if store_labels: - self.label_list = [load_label(p, inds, tot) for p in label_paths] - else: - self.label_paths = label_paths - self.label_offsets_list = [ - load_label_offset(p, inds, tot) for p in label_paths - ] - assert label_processors is None or len(label_processors) == self.num_labels - for label_path, label_rate in zip(label_paths, self.label_rates): - verify_label_lengths( - self.sizes, sample_rate, label_path, label_rate, inds, tot - ) - - self.max_sample_size = ( - max_sample_size if max_sample_size is not None else sys.maxsize - ) - self.pad_audio = pad_audio - self.normalize = normalize - logger.info( - f"pad_audio={pad_audio}, random_crop={random_crop}, " - f"normalize={normalize}, max_sample_size={self.max_sample_size}" - ) - - def get_audio(self, index): - import soundfile as sf - - wav_path = os.path.join(self.audio_root, self.audio_names[index]) - wav, cur_sample_rate = sf.read(wav_path) - wav = torch.from_numpy(wav).float() - wav = self.postprocess(wav, cur_sample_rate) - return wav - - def get_label(self, index, label_idx): - if self.store_labels: - label = self.label_list[label_idx][index] - else: - with open(self.label_paths[label_idx]) as f: - offset_s, offset_e = self.label_offsets_list[label_idx][index] - f.seek(offset_s) - label = f.read(offset_e - offset_s) - - if self.label_processors is not None: - label = self.label_processors[label_idx](label) - return label - - def get_labels(self, index): - return [self.get_label(index, i) for i in range(self.num_labels)] - - def __getitem__(self, index): - wav = self.get_audio(index) - labels = self.get_labels(index) - return {"id": index, "source": wav, "label_list": labels} - - def __len__(self): - return len(self.sizes) - - def crop_to_max_size(self, wav, target_size): - size = len(wav) - diff = size - target_size - if diff <= 0: - return wav, 0 - - start, end = 0, target_size - if self.random_crop: - start = np.random.randint(0, diff + 1) - end = size - diff + start - return wav[start:end], start - - def collater(self, samples): - # target = max(sizes) -> random_crop not used - # target = max_sample_size -> random_crop used for long - samples = [s for s in samples if s["source"] is not None] - if len(samples) == 0: - return {} - - audios = [s["source"] for s in samples] - audio_sizes = [len(s) for s in audios] - if self.pad_audio: - audio_size = min(max(audio_sizes), self.max_sample_size) - else: - audio_size = min(min(audio_sizes), self.max_sample_size) - collated_audios, padding_mask, audio_starts = self.collater_audio( - audios, audio_size - ) - - targets_by_label = [ - [s["label_list"][i] for s in samples] for i in range(self.num_labels) - ] - targets_list, lengths_list, ntokens_list = self.collater_label( - targets_by_label, audio_size, audio_starts - ) - - net_input = {"source": collated_audios, "padding_mask": padding_mask} - batch = { - "id": torch.LongTensor([s["id"] for s in samples]), - "net_input": net_input, - } - - if self.single_target: - batch["target_lengths"] = lengths_list[0] - batch["ntokens"] = ntokens_list[0] - batch["target"] = targets_list[0] - else: - batch["target_lengths_list"] = lengths_list - batch["ntokens_list"] = ntokens_list - batch["target_list"] = targets_list - return batch - - def collater_audio(self, audios, audio_size): - collated_audios = audios[0].new_zeros(len(audios), audio_size) - padding_mask = ( - torch.BoolTensor(collated_audios.shape).fill_(False) - # if self.pad_audio else None - ) - audio_starts = [0 for _ in audios] - for i, audio in enumerate(audios): - diff = len(audio) - audio_size - if diff == 0: - collated_audios[i] = audio - elif diff < 0: - assert self.pad_audio - collated_audios[i] = torch.cat([audio, audio.new_full((-diff,), 0.0)]) - padding_mask[i, diff:] = True - else: - collated_audios[i], audio_starts[i] = self.crop_to_max_size( - audio, audio_size - ) - return collated_audios, padding_mask, audio_starts - - def collater_frm_label(self, targets, audio_size, audio_starts, label_rate, pad): - assert label_rate > 0 - s2f = label_rate / self.sample_rate - frm_starts = [int(round(s * s2f)) for s in audio_starts] - frm_size = int(round(audio_size * s2f)) - if not self.pad_audio: - rem_size = [len(t) - s for t, s in zip(targets, frm_starts)] - frm_size = min(frm_size, *rem_size) - targets = [t[s : s + frm_size] for t, s in zip(targets, frm_starts)] - logger.debug(f"audio_starts={audio_starts}") - logger.debug(f"frame_starts={frm_starts}") - logger.debug(f"frame_size={frm_size}") - - lengths = torch.LongTensor([len(t) for t in targets]) - ntokens = lengths.sum().item() - targets = data_utils.collate_tokens(targets, pad_idx=pad, left_pad=False) - return targets, lengths, ntokens - - def collater_seq_label(self, targets, pad): - lengths = torch.LongTensor([len(t) for t in targets]) - ntokens = lengths.sum().item() - targets = data_utils.collate_tokens(targets, pad_idx=pad, left_pad=False) - return targets, lengths, ntokens - - def collater_label(self, targets_by_label, audio_size, audio_starts): - targets_list, lengths_list, ntokens_list = [], [], [] - itr = zip(targets_by_label, self.label_rates, self.pad_list) - for targets, label_rate, pad in itr: - if label_rate == -1.0: - targets, lengths, ntokens = self.collater_seq_label(targets, pad) - else: - targets, lengths, ntokens = self.collater_frm_label( - targets, audio_size, audio_starts, label_rate, pad - ) - targets_list.append(targets) - lengths_list.append(lengths) - ntokens_list.append(ntokens) - return targets_list, lengths_list, ntokens_list - - def num_tokens(self, index): - return self.size(index) - - def size(self, index): - if self.pad_audio: - return self.sizes[index] - return min(self.sizes[index], self.max_sample_size) - - def ordered_indices(self): - if self.shuffle: - order = [np.random.permutation(len(self))] - else: - order = [np.arange(len(self))] - - order.append(self.sizes) - return np.lexsort(order)[::-1] - - def postprocess(self, wav, cur_sample_rate): - if wav.dim() == 2: - wav = wav.mean(-1) - assert wav.dim() == 1, wav.dim() - - if cur_sample_rate != self.sample_rate: - raise Exception(f"sr {cur_sample_rate} != {self.sample_rate}") - - if self.normalize: - with torch.no_grad(): - wav = F.layer_norm(wav, wav.shape) - return wav diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/multi_modality_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/multi_modality_dataset.py deleted file mode 100644 index 940906507be4a106d4a0c69ca400e65b374c0a76..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/multi_modality_dataset.py +++ /dev/null @@ -1,266 +0,0 @@ -# Copyright (c) 2021-present, Facebook, Inc. -# All rights reserved. -# -# This source code is licensed under the license found in the LICENSE file in -# the root directory of this source tree. An additional grant of patent rights -# can be found in the PATENTS file in the same directory. - -import logging -import math -from typing import List, NamedTuple, Optional - -import numpy as np -import torch -from fairseq.data import ( - ConcatDataset, - FairseqDataset, - FileAudioDataset, - LanguagePairDataset, - data_utils, -) - -logger = logging.getLogger(__name__) - - -class ModalityDatasetItem(NamedTuple): - datasetname: str - dataset: any - max_positions: List[int] - max_tokens: Optional[int] = None - max_sentences: Optional[int] = None - - -# MultiModalityDataset: it concate multiple datasets with different modalities. -# Compared with ConcatDataset it can 1) sample data given the ratios for different datasets -# 2) it adds mode to indicate what type of the data samples come from. -# It will be used with GroupedEpochBatchIterator together to generate mini-batch with samples -# from the same type of dataset -# If only one dataset is used, it will perform like the original dataset with mode added -class MultiModalityDataset(ConcatDataset): - def __init__(self, datasets: List[ModalityDatasetItem]): - id_to_mode = [] - dsets = [] - max_tokens = [] - max_sentences = [] - max_positions = [] - for dset in datasets: - id_to_mode.append(dset.datasetname) - dsets.append(dset.dataset) - max_tokens.append(dset.max_tokens) - max_positions.append(dset.max_positions) - max_sentences.append(dset.max_sentences) - weights = [1.0 for s in dsets] - super().__init__(dsets, weights) - self.max_tokens = max_tokens - self.max_positions = max_positions - self.max_sentences = max_sentences - self.id_to_mode = id_to_mode - self.raw_sub_batch_samplers = [] - self._cur_epoch = 0 - - def set_epoch(self, epoch): - super().set_epoch(epoch) - self._cur_epoch = epoch - - def __getitem__(self, idx): - dataset_idx, sample_idx = self._get_dataset_and_sample_index(idx) - sample = self.datasets[dataset_idx][sample_idx] - return (dataset_idx, sample) - - def collater(self, samples): - if len(samples) == 0: - return {} - dataset_idx = samples[0][0] - # make sure all samples in samples are from same dataset - assert sum([0 if dataset_idx == s[0] else 1 for s in samples]) == 0 - samples = self.datasets[dataset_idx].collater([x[1] for x in samples]) - # add mode - samples["net_input"]["mode"] = self.id_to_mode[dataset_idx] - - return samples - - def size(self, index: int): - if len(self.datasets) == 1: - return self.datasets[0].size(index) - return super().size(index) - - @property - def sizes(self): - if len(self.datasets) == 1: - return self.datasets[0].sizes - super().sizes - - def ordered_indices(self): - """ - Returns indices sorted by length. So less padding is needed. - """ - if len(self.datasets) == 1: - return self.datasets[0].ordered_indices() - indices_group = [] - for d_idx, ds in enumerate(self.datasets): - sample_num = self.cumulative_sizes[d_idx] - if d_idx > 0: - sample_num = sample_num - self.cumulative_sizes[d_idx - 1] - assert sample_num == len(ds) - indices_group.append(ds.ordered_indices()) - return indices_group - - def get_raw_batch_samplers(self, required_batch_size_multiple, seed): - if len(self.raw_sub_batch_samplers) > 0: - logger.info(" raw_sub_batch_samplers exists. No action is taken") - return - with data_utils.numpy_seed(seed): - indices = self.ordered_indices() - for i, ds in enumerate(self.datasets): - indices[i] = ds.filter_indices_by_size( - indices[i], - self.max_positions[i], - )[0] - sub_batch_sampler = ds.batch_by_size( - indices[i], - max_tokens=self.max_tokens[i], - max_sentences=self.max_sentences[i], - required_batch_size_multiple=required_batch_size_multiple, - ) - self.raw_sub_batch_samplers.append(sub_batch_sampler) - - def get_batch_samplers(self, mult_ratios, required_batch_size_multiple, seed): - self.get_raw_batch_samplers(required_batch_size_multiple, seed) - batch_samplers = [] - for i, _ in enumerate(self.datasets): - if i > 0: - sub_batch_sampler = [ - [y + self.cumulative_sizes[i - 1] for y in x] - for x in self.raw_sub_batch_samplers[i] - ] - else: - sub_batch_sampler = list(self.raw_sub_batch_samplers[i]) - smp_r = mult_ratios[i] - if smp_r != 1: - is_increase = "increased" if smp_r > 1 else "decreased" - logger.info( - "number of batch for the dataset {} is {} from {} to {}".format( - self.id_to_mode[i], - is_increase, - len(sub_batch_sampler), - int(len(sub_batch_sampler) * smp_r), - ) - ) - mul_samplers = [] - for _ in range(math.floor(smp_r)): - mul_samplers = mul_samplers + sub_batch_sampler - if math.floor(smp_r) != smp_r: - with data_utils.numpy_seed(seed + self._cur_epoch): - np.random.shuffle(sub_batch_sampler) - smp_num = int( - (smp_r - math.floor(smp_r)) * len(sub_batch_sampler) - ) - mul_samplers = mul_samplers + sub_batch_sampler[:smp_num] - sub_batch_sampler = mul_samplers - else: - logger.info( - "dataset {} batch number is {} ".format( - self.id_to_mode[i], len(sub_batch_sampler) - ) - ) - batch_samplers.append(sub_batch_sampler) - - return batch_samplers - - -class LangPairMaskDataset(FairseqDataset): - def __init__( - self, - dataset: LanguagePairDataset, - src_eos: int, - src_bos: Optional[int] = None, - noise_id: Optional[int] = -1, - mask_ratio: Optional[float] = 0, - mask_type: Optional[str] = "random", - ): - self.dataset = dataset - self.src_eos = src_eos - self.src_bos = src_bos - self.noise_id = noise_id - self.mask_ratio = mask_ratio - self.mask_type = mask_type - assert mask_type in ("random", "tail") - - @property - def src_sizes(self): - return self.dataset.src_sizes - - @property - def tgt_sizes(self): - return self.dataset.tgt_sizes - - @property - def sizes(self): - # dataset.sizes can be a dynamically computed sizes: - return self.dataset.sizes - - def get_batch_shapes(self): - if hasattr(self.dataset, "get_batch_shapes"): - return self.dataset.get_batch_shapes() - return self.dataset.buckets - - def num_tokens_vec(self, indices): - return self.dataset.num_tokens_vec(indices) - - def __len__(self): - return len(self.dataset) - - def num_tokens(self, index): - return self.dataset.num_tokens(index) - - def size(self, index): - return self.dataset.size(index) - - def ordered_indices(self): - return self.dataset.ordered_indices() - - @property - def supports_prefetch(self): - return getattr(self.dataset, "supports_prefetch", False) - - def prefetch(self, indices): - return self.dataset.prefetch(indices) - - def mask_src_tokens(self, sample): - src_item = sample["source"] - mask = None - if self.mask_type == "random": - mask = torch.rand(len(src_item)).le(self.mask_ratio) - else: - mask = torch.ones(len(src_item)) - mask[: int(len(src_item) * (1 - self.mask_ratio))] = 0 - mask = mask.eq(1) - if src_item[0] == self.src_bos: - mask[0] = False - if src_item[-1] == self.src_eos: - mask[-1] = False - mask_src_item = src_item.masked_fill(mask, self.noise_id) - smp = {"id": sample["id"], "source": mask_src_item, "target": sample["target"]} - return smp - - def __getitem__(self, index): - sample = self.dataset[index] - if self.mask_ratio > 0: - sample = self.mask_src_tokens(sample) - return sample - - def collater(self, samples, pad_to_length=None): - return self.dataset.collater(samples, pad_to_length) - - -class FileAudioDatasetWrapper(FileAudioDataset): - def collater(self, samples): - samples = super().collater(samples) - if len(samples) == 0: - return {} - samples["net_input"]["src_tokens"] = samples["net_input"]["source"] - samples["net_input"]["prev_output_tokens"] = None - del samples["net_input"]["source"] - samples["net_input"]["src_lengths"] = None - samples["net_input"]["alignment"] = None - return samples diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/raw_audio_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/raw_audio_dataset.py deleted file mode 100644 index 5c01325d585972abe8f5a2bad182e9f9454e5c33..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/raw_audio_dataset.py +++ /dev/null @@ -1,392 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - - -import io -import logging -import os -import sys - -import numpy as np -import torch -import torch.nn.functional as F -from fairseq.data.audio.audio_utils import ( - is_sf_audio_data, - parse_path, - read_from_stored_zip, -) -from fairseq.data.text_compressor import TextCompressionLevel, TextCompressor - -from .. import FairseqDataset -from ..data_utils import compute_mask_indices, get_bucketed_sizes, get_buckets - -logger = logging.getLogger(__name__) - - -class RawAudioDataset(FairseqDataset): - def __init__( - self, - sample_rate, - max_sample_size=None, - min_sample_size=0, - shuffle=True, - pad=False, - normalize=False, - compute_mask_indices=False, - **mask_compute_kwargs, - ): - super().__init__() - - self.sample_rate = sample_rate - self.sizes = [] - self.max_sample_size = ( - max_sample_size if max_sample_size is not None else sys.maxsize - ) - self.min_sample_size = min_sample_size - self.pad = pad - self.shuffle = shuffle - self.normalize = normalize - self.compute_mask_indices = compute_mask_indices - if self.compute_mask_indices: - self.mask_compute_kwargs = mask_compute_kwargs - self._features_size_map = {} - self._C = mask_compute_kwargs["encoder_embed_dim"] - self._conv_feature_layers = eval(mask_compute_kwargs["conv_feature_layers"]) - - def __getitem__(self, index): - raise NotImplementedError() - - def __len__(self): - return len(self.sizes) - - def postprocess(self, feats, curr_sample_rate): - if feats.dim() == 2: - feats = feats.mean(-1) - - if curr_sample_rate != self.sample_rate: - raise Exception(f"sample rate: {curr_sample_rate}, need {self.sample_rate}") - - assert feats.dim() == 1, feats.dim() - - if self.normalize: - with torch.no_grad(): - feats = F.layer_norm(feats, feats.shape) - return feats - - def crop_to_max_size(self, wav, target_size): - size = len(wav) - diff = size - target_size - if diff <= 0: - return wav - - start = np.random.randint(0, diff + 1) - end = size - diff + start - return wav[start:end] - - def _compute_mask_indices(self, dims, padding_mask): - B, T, C = dims - mask_indices, mask_channel_indices = None, None - if self.mask_compute_kwargs["mask_prob"] > 0: - mask_indices = compute_mask_indices( - (B, T), - padding_mask, - self.mask_compute_kwargs["mask_prob"], - self.mask_compute_kwargs["mask_length"], - self.mask_compute_kwargs["mask_selection"], - self.mask_compute_kwargs["mask_other"], - min_masks=2, - no_overlap=self.mask_compute_kwargs["no_mask_overlap"], - min_space=self.mask_compute_kwargs["mask_min_space"], - ) - mask_indices = torch.from_numpy(mask_indices) - if self.mask_compute_kwargs["mask_channel_prob"] > 0: - mask_channel_indices = compute_mask_indices( - (B, C), - None, - self.mask_compute_kwargs["mask_channel_prob"], - self.mask_compute_kwargs["mask_channel_length"], - self.mask_compute_kwargs["mask_channel_selection"], - self.mask_compute_kwargs["mask_channel_other"], - no_overlap=self.mask_compute_kwargs["no_mask_channel_overlap"], - min_space=self.mask_compute_kwargs["mask_channel_min_space"], - ) - mask_channel_indices = ( - torch.from_numpy(mask_channel_indices).unsqueeze(1).expand(-1, T, -1) - ) - - return mask_indices, mask_channel_indices - - @staticmethod - def _bucket_tensor(tensor, num_pad, value): - return F.pad(tensor, (0, num_pad), value=value) - - def collater(self, samples): - samples = [s for s in samples if s["source"] is not None] - if len(samples) == 0: - return {} - - sources = [s["source"] for s in samples] - sizes = [len(s) for s in sources] - - if self.pad: - target_size = min(max(sizes), self.max_sample_size) - else: - target_size = min(min(sizes), self.max_sample_size) - - collated_sources = sources[0].new_zeros(len(sources), target_size) - padding_mask = ( - torch.BoolTensor(collated_sources.shape).fill_(False) if self.pad else None - ) - for i, (source, size) in enumerate(zip(sources, sizes)): - diff = size - target_size - if diff == 0: - collated_sources[i] = source - elif diff < 0: - assert self.pad - collated_sources[i] = torch.cat( - [source, source.new_full((-diff,), 0.0)] - ) - padding_mask[i, diff:] = True - else: - collated_sources[i] = self.crop_to_max_size(source, target_size) - - input = {"source": collated_sources} - out = {"id": torch.LongTensor([s["id"] for s in samples])} - if self.pad: - input["padding_mask"] = padding_mask - - if hasattr(self, "num_buckets") and self.num_buckets > 0: - assert self.pad, "Cannot bucket without padding first." - bucket = max(self._bucketed_sizes[s["id"]] for s in samples) - num_pad = bucket - collated_sources.size(-1) - if num_pad: - input["source"] = self._bucket_tensor(collated_sources, num_pad, 0) - input["padding_mask"] = self._bucket_tensor(padding_mask, num_pad, True) - - if self.compute_mask_indices: - B = input["source"].size(0) - T = self._get_mask_indices_dims(input["source"].size(-1)) - padding_mask_reshaped = input["padding_mask"].clone() - extra = padding_mask_reshaped.size(1) % T - if extra > 0: - padding_mask_reshaped = padding_mask_reshaped[:, :-extra] - padding_mask_reshaped = padding_mask_reshaped.view( - padding_mask_reshaped.size(0), T, -1 - ) - padding_mask_reshaped = padding_mask_reshaped.all(-1) - input["padding_count"] = padding_mask_reshaped.sum(-1).max().item() - mask_indices, mask_channel_indices = self._compute_mask_indices( - (B, T, self._C), - padding_mask_reshaped, - ) - input["mask_indices"] = mask_indices - input["mask_channel_indices"] = mask_channel_indices - out["sample_size"] = mask_indices.sum().item() - - out["net_input"] = input - return out - - def _get_mask_indices_dims(self, size, padding=0, dilation=1): - if size not in self._features_size_map: - L_in = size - for (_, kernel_size, stride) in self._conv_feature_layers: - L_out = L_in + 2 * padding - dilation * (kernel_size - 1) - 1 - L_out = 1 + L_out // stride - L_in = L_out - self._features_size_map[size] = L_out - return self._features_size_map[size] - - def num_tokens(self, index): - return self.size(index) - - def size(self, index): - """Return an example's size as a float or tuple. This value is used when - filtering a dataset with ``--max-positions``.""" - if self.pad: - return self.sizes[index] - return min(self.sizes[index], self.max_sample_size) - - def ordered_indices(self): - """Return an ordered list of indices. Batches will be constructed based - on this order.""" - - if self.shuffle: - order = [np.random.permutation(len(self))] - order.append( - np.minimum( - np.array(self.sizes), - self.max_sample_size, - ) - ) - return np.lexsort(order)[::-1] - else: - return np.arange(len(self)) - - def set_bucket_info(self, num_buckets): - self.num_buckets = num_buckets - if self.num_buckets > 0: - self._collated_sizes = np.minimum( - np.array(self.sizes), - self.max_sample_size, - ) - self.buckets = get_buckets( - self._collated_sizes, - self.num_buckets, - ) - self._bucketed_sizes = get_bucketed_sizes( - self._collated_sizes, self.buckets - ) - logger.info( - f"{len(self.buckets)} bucket(s) for the audio dataset: " - f"{self.buckets}" - ) - - -class FileAudioDataset(RawAudioDataset): - def __init__( - self, - manifest_path, - sample_rate, - max_sample_size=None, - min_sample_size=0, - shuffle=True, - pad=False, - normalize=False, - num_buckets=0, - compute_mask_indices=False, - text_compression_level=TextCompressionLevel.none, - **mask_compute_kwargs, - ): - super().__init__( - sample_rate=sample_rate, - max_sample_size=max_sample_size, - min_sample_size=min_sample_size, - shuffle=shuffle, - pad=pad, - normalize=normalize, - compute_mask_indices=compute_mask_indices, - **mask_compute_kwargs, - ) - - self.text_compressor = TextCompressor(level=text_compression_level) - - skipped = 0 - self.fnames = [] - sizes = [] - self.skipped_indices = set() - - with open(manifest_path, "r") as f: - self.root_dir = f.readline().strip() - for i, line in enumerate(f): - items = line.strip().split("\t") - assert len(items) == 2, line - sz = int(items[1]) - if min_sample_size is not None and sz < min_sample_size: - skipped += 1 - self.skipped_indices.add(i) - continue - self.fnames.append(self.text_compressor.compress(items[0])) - sizes.append(sz) - logger.info(f"loaded {len(self.fnames)}, skipped {skipped} samples") - - self.sizes = np.array(sizes, dtype=np.int64) - - try: - import pyarrow - - self.fnames = pyarrow.array(self.fnames) - except: - logger.debug( - "Could not create a pyarrow array. Please install pyarrow for better performance" - ) - pass - - self.set_bucket_info(num_buckets) - - def __getitem__(self, index): - import soundfile as sf - - fn = self.fnames[index] - fn = fn if isinstance(self.fnames, list) else fn.as_py() - fn = self.text_compressor.decompress(fn) - path_or_fp = os.path.join(self.root_dir, fn) - _path, slice_ptr = parse_path(path_or_fp) - if len(slice_ptr) == 2: - byte_data = read_from_stored_zip(_path, slice_ptr[0], slice_ptr[1]) - assert is_sf_audio_data(byte_data) - path_or_fp = io.BytesIO(byte_data) - - wav, curr_sample_rate = sf.read(path_or_fp, dtype="float32") - - feats = torch.from_numpy(wav).float() - feats = self.postprocess(feats, curr_sample_rate) - return {"id": index, "source": feats} - - -class BinarizedAudioDataset(RawAudioDataset): - def __init__( - self, - data_dir, - split, - sample_rate, - max_sample_size=None, - min_sample_size=0, - shuffle=True, - pad=False, - normalize=False, - num_buckets=0, - compute_mask_indices=False, - **mask_compute_kwargs, - ): - super().__init__( - sample_rate=sample_rate, - max_sample_size=max_sample_size, - min_sample_size=min_sample_size, - shuffle=shuffle, - pad=pad, - normalize=normalize, - compute_mask_indices=compute_mask_indices, - **mask_compute_kwargs, - ) - - from fairseq.data import Dictionary, data_utils - - self.fnames_dict = Dictionary.load(os.path.join(data_dir, "dict.txt")) - - root_path = os.path.join(data_dir, f"{split}.root") - if os.path.exists(root_path): - with open(root_path, "r") as f: - self.root_dir = next(f).strip() - else: - self.root_dir = None - - fnames_path = os.path.join(data_dir, split) - self.fnames = data_utils.load_indexed_dataset(fnames_path, self.fnames_dict) - lengths_path = os.path.join(data_dir, f"{split}.lengths") - - with open(lengths_path, "r") as f: - for line in f: - sz = int(line.rstrip()) - assert ( - sz >= min_sample_size - ), f"Min sample size is not supported for binarized dataset, but found a sample with size {sz}" - self.sizes.append(sz) - - self.sizes = np.array(self.sizes, dtype=np.int64) - - self.set_bucket_info(num_buckets) - logger.info(f"loaded {len(self.fnames)} samples") - - def __getitem__(self, index): - import soundfile as sf - - fname = self.fnames_dict.string(self.fnames[index], separator="") - if self.root_dir: - fname = os.path.join(self.root_dir, fname) - - wav, curr_sample_rate = sf.read(fname) - feats = torch.from_numpy(wav).float() - feats = self.postprocess(feats, curr_sample_rate) - return {"id": index, "source": feats} diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/speech_to_speech_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/speech_to_speech_dataset.py deleted file mode 100644 index 641e3b108f0343852a76c43963a92dc367bf87fb..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/speech_to_speech_dataset.py +++ /dev/null @@ -1,427 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import logging -from dataclasses import dataclass -from pathlib import Path -from typing import Dict, List, Optional, Tuple - -import torch -from fairseq.data import ConcatDataset, Dictionary -from fairseq.data import data_utils as fairseq_data_utils -from fairseq.data.audio.data_cfg import S2SDataConfig -from fairseq.data.audio.speech_to_text_dataset import ( - SpeechToTextDataset, - SpeechToTextDatasetCreator, - _collate_frames, - get_features_or_waveform, -) - -logger = logging.getLogger(__name__) - - -@dataclass -class SpeechToSpeechDatasetItem(object): - index: int - source: torch.Tensor - target: Optional[torch.Tensor] = None - target_speaker: Optional[torch.Tensor] = None - tgt_lang_tag: Optional[int] = None - - -class SpeechToSpeechDataset(SpeechToTextDataset): - def __init__( - self, - split: str, - is_train_split: bool, - data_cfg: S2SDataConfig, - src_audio_paths: List[str], - src_n_frames: List[int], - tgt_audio_paths: List[str], - tgt_n_frames: List[int], - src_langs: Optional[List[str]] = None, - tgt_langs: Optional[List[str]] = None, - ids: Optional[List[str]] = None, - target_is_code: bool = False, - tgt_dict: Dictionary = None, - n_frames_per_step: int = 1, - ): - tgt_texts = tgt_audio_paths if target_is_code else None - super().__init__( - split, - is_train_split, - data_cfg, - src_audio_paths, - src_n_frames, - ids=ids, - tgt_dict=tgt_dict, - tgt_texts=tgt_texts, - src_langs=src_langs, - tgt_langs=tgt_langs, - n_frames_per_step=n_frames_per_step, - ) - - self.tgt_audio_paths = tgt_audio_paths - self.tgt_lens = [t // self.n_frames_per_step for t in tgt_n_frames] - - assert not target_is_code or tgt_dict is not None - self.target_is_code = target_is_code - - assert len(tgt_audio_paths) == self.n_samples - assert len(tgt_n_frames) == self.n_samples - - self.tgt_speakers = None - if self.cfg.target_speaker_embed: - samples = SpeechToTextDatasetCreator._load_samples_from_tsv( - self.cfg.target_speaker_embed, split - ) - spk_emb_dict = {s["id"]: s["speaker_embed"] for s in samples} - self.tgt_speakers = [spk_emb_dict[id] for id in self.ids] - assert len(self.tgt_speakers) == self.n_samples - - logger.info(self.__repr__()) - - def pack_units(self, input: torch.Tensor) -> torch.Tensor: - if self.n_frames_per_step <= 1: - return input - - offset = 4 - vocab_size = ( - len(self.tgt_dict) - offset - ) # remove offset from <bos>, <pad>, <eos>, <unk>, which is specific to fairseq dictionary - - assert input.dim() == 1 - stacked_input = ( - input[:-1].view(-1, self.n_frames_per_step) - offset - ) # remove <eos> - scale = [ - pow(vocab_size, self.n_frames_per_step - 1 - i) - for i in range(self.n_frames_per_step) - ] - scale = torch.LongTensor(scale).squeeze(0) - res = input.new((len(input) - 1) // self.n_frames_per_step + 1).fill_(input[-1]) - res[:-1] = (stacked_input * scale).sum(dim=1) + offset - - return res - - def __getitem__(self, index: int) -> SpeechToSpeechDatasetItem: - source = self._get_source_audio(index) - - tgt_lang_tag = None - if self.cfg.prepend_tgt_lang_tag_as_bos: - # prepend_tgt_lang_tag_as_bos: put tgt_lang_tag as bos of target - tgt_lang_tag = self.get_lang_tag_idx(self.tgt_langs[index], self.tgt_dict) - - if not self.target_is_code: - target = get_features_or_waveform(self.tgt_audio_paths[index]) - target = torch.from_numpy(target).float() - target = self.pack_frames(target) - else: - target = self.tgt_dict.encode_line( - self.tgt_audio_paths[index], - add_if_not_exist=False, - append_eos=True, - ).long() - if self.n_frames_per_step > 1: - n_tgt_frame = target.size(0) - 1 # exclude <eos> - keep_n_tgt_frame = n_tgt_frame - n_tgt_frame % self.n_frames_per_step - target = torch.cat( - ( - target[:keep_n_tgt_frame], - target.new_full((1,), self.tgt_dict.eos()), - ), - dim=0, - ) - - if self.tgt_speakers: - tgt_spk = get_features_or_waveform(self.tgt_speakers[index]) - tgt_spk = torch.from_numpy(tgt_spk).float() - else: - tgt_spk = torch.FloatTensor([]) - - return SpeechToSpeechDatasetItem( - index=index, - source=source, - target=target, - target_speaker=tgt_spk, - tgt_lang_tag=tgt_lang_tag, - ) - - def _collate_target(self, samples: List[SpeechToSpeechDatasetItem]) -> torch.Tensor: - if self.target_is_code: - target = fairseq_data_utils.collate_tokens( - [x.target for x in samples], - self.tgt_dict.pad(), - self.tgt_dict.eos(), - left_pad=False, - move_eos_to_beginning=False, - ) - # convert stacked units to a single id - pack_targets = [self.pack_units(x.target) for x in samples] - prev_output_tokens = fairseq_data_utils.collate_tokens( - pack_targets, - self.tgt_dict.pad(), - self.tgt_dict.eos(), - left_pad=False, - move_eos_to_beginning=True, - ) - target_lengths = torch.tensor( - [x.size(0) for x in pack_targets], dtype=torch.long - ) - else: - target = _collate_frames([x.target for x in samples], is_audio_input=False) - bsz, _, d = target.size() - prev_output_tokens = torch.cat( - (target.new_full((bsz, 1, d), 0.0), target[:, :-1, :]), dim=1 - ) - target_lengths = torch.tensor( - [x.target.size(0) for x in samples], dtype=torch.long - ) - - return target, prev_output_tokens, target_lengths - - def collater( - self, samples: List[SpeechToSpeechDatasetItem], return_order: bool = False - ) -> Dict: - if len(samples) == 0: - return {} - indices = torch.tensor([x.index for x in samples], dtype=torch.long) - frames = _collate_frames([x.source for x in samples], self.cfg.use_audio_input) - # sort samples by descending number of frames - n_frames = torch.tensor([x.source.size(0) for x in samples], dtype=torch.long) - n_frames, order = n_frames.sort(descending=True) - indices = indices.index_select(0, order) - frames = frames.index_select(0, order) - - target, prev_output_tokens, target_lengths = self._collate_target(samples) - target = target.index_select(0, order) - target_lengths = target_lengths.index_select(0, order) - prev_output_tokens = prev_output_tokens.index_select(0, order) - ntokens = sum(x.target.size(0) for x in samples) - - tgt_speakers = None - if self.cfg.target_speaker_embed: - tgt_speakers = _collate_frames( - [x.target_speaker for x in samples], is_audio_input=True - ).index_select(0, order) - - net_input = { - "src_tokens": frames, - "src_lengths": n_frames, - "prev_output_tokens": prev_output_tokens, - "tgt_speaker": tgt_speakers, # TODO: unify "speaker" and "tgt_speaker" - } - if self.tgt_texts is not None and samples[0].tgt_lang_tag is not None: - for i in range(len(samples)): - net_input["prev_output_tokens"][i][0] = samples[order[i]].tgt_lang_tag - out = { - "id": indices, - "net_input": net_input, - "speaker": tgt_speakers, # to support Tacotron2 loss for speech-to-spectrogram model - "target": target, - "target_lengths": target_lengths, - "ntokens": ntokens, - "nsentences": len(samples), - } - if return_order: - out["order"] = order - return out - - -class TextTargetMultitaskData(object): - # mandatory columns - KEY_ID, KEY_TEXT = "id", "tgt_text" - - def __init__(self, args, split, tgt_dict): - samples = SpeechToTextDatasetCreator._load_samples_from_tsv(args.data, split) - self.data = {s[self.KEY_ID]: s[self.KEY_TEXT] for s in samples} - self.dict = tgt_dict - self.append_eos = args.decoder_type != "ctc" - - def get(self, sample_id): - if sample_id in self.data: - return self.dict.encode_line( - self.data[sample_id], - add_if_not_exist=False, - append_eos=self.append_eos, - ) - else: - logger.warning(f"no target for {sample_id}") - return torch.IntTensor([]) - - def collater(self, samples: List[torch.Tensor]) -> torch.Tensor: - out = fairseq_data_utils.collate_tokens( - samples, - self.dict.pad(), - self.dict.eos(), - left_pad=False, - move_eos_to_beginning=False, - ).long() - - prev_out = fairseq_data_utils.collate_tokens( - samples, - self.dict.pad(), - self.dict.eos(), - left_pad=False, - move_eos_to_beginning=True, - ).long() - - target_lengths = torch.tensor([t.size(0) for t in samples], dtype=torch.long) - ntokens = sum(t.size(0) for t in samples) - - output = { - "prev_output_tokens": prev_out, - "target": out, - "target_lengths": target_lengths, - "ntokens": ntokens, - } - - return output - - -class SpeechToSpeechMultitaskDataset(SpeechToSpeechDataset): - def __init__(self, *argv): - super().__init__(*argv) - self.multitask_data = {} - - def add_multitask_dataset(self, task_name, task_data): - self.multitask_data[task_name] = task_data - - def __getitem__( - self, index: int - ) -> Tuple[SpeechToSpeechDatasetItem, Dict[str, torch.Tensor]]: - s2s_data = super().__getitem__(index) - - multitask_target = {} - sample_id = self.ids[index] - for task_name, task_dataset in self.multitask_data.items(): - multitask_target[task_name] = task_dataset.get(sample_id) - - return s2s_data, multitask_target - - def collater( - self, samples: List[Tuple[SpeechToSpeechDatasetItem, Dict[str, torch.Tensor]]] - ) -> Dict: - if len(samples) == 0: - return {} - - out = super().collater([s for s, _ in samples], return_order=True) - order = out["order"] - del out["order"] - - for task_name, task_dataset in self.multitask_data.items(): - if "multitask" not in out: - out["multitask"] = {} - d = [s[task_name] for _, s in samples] - task_target = task_dataset.collater(d) - out["multitask"][task_name] = { - "target": task_target["target"].index_select(0, order), - "target_lengths": task_target["target_lengths"].index_select(0, order), - "ntokens": task_target["ntokens"], - } - out["multitask"][task_name]["net_input"] = { - "prev_output_tokens": task_target["prev_output_tokens"].index_select( - 0, order - ), - } - - return out - - -class SpeechToSpeechDatasetCreator(object): - # mandatory columns - KEY_ID, KEY_SRC_AUDIO, KEY_SRC_N_FRAMES = "id", "src_audio", "src_n_frames" - KEY_TGT_AUDIO, KEY_TGT_N_FRAMES = "tgt_audio", "tgt_n_frames" - # optional columns - KEY_SRC_LANG, KEY_TGT_LANG = "src_lang", "tgt_lang" - # default values - DEFAULT_LANG = "" - - @classmethod - def _from_list( - cls, - split_name: str, - is_train_split, - samples: List[Dict], - data_cfg: S2SDataConfig, - target_is_code: bool = False, - target_dictionary: Dictionary = None, - n_frames_per_step: int = 1, - multitask: Optional[Dict] = None, - ) -> SpeechToSpeechDataset: - audio_root = Path(data_cfg.audio_root) - ids = [s[cls.KEY_ID] for s in samples] - src_audio_paths = [ - (audio_root / s[cls.KEY_SRC_AUDIO]).as_posix() for s in samples - ] - tgt_audio_paths = [ - s[cls.KEY_TGT_AUDIO] - if target_is_code - else (audio_root / s[cls.KEY_TGT_AUDIO]).as_posix() - for s in samples - ] - src_n_frames = [int(s[cls.KEY_SRC_N_FRAMES]) for s in samples] - tgt_n_frames = [int(s[cls.KEY_TGT_N_FRAMES]) for s in samples] - src_langs = [s.get(cls.KEY_SRC_LANG, cls.DEFAULT_LANG) for s in samples] - tgt_langs = [s.get(cls.KEY_TGT_LANG, cls.DEFAULT_LANG) for s in samples] - - has_multitask = len(multitask) > 0 - dataset_cls = ( - SpeechToSpeechMultitaskDataset if has_multitask else SpeechToSpeechDataset - ) - - ds = dataset_cls( - split_name, - is_train_split, - data_cfg, - src_audio_paths, - src_n_frames, - tgt_audio_paths, - tgt_n_frames, - src_langs, - tgt_langs, - ids, - target_is_code, - target_dictionary, - n_frames_per_step, - ) - - if has_multitask: - for task_name, task_obj in multitask.items(): - task_data = TextTargetMultitaskData( - task_obj.args, split_name, task_obj.target_dictionary - ) - ds.add_multitask_dataset(task_name, task_data) - return ds - - @classmethod - def from_tsv( - cls, - root: str, - data_cfg: S2SDataConfig, - splits: str, - is_train_split: bool, - epoch: int, - seed: int, - target_is_code: bool = False, - target_dictionary: Dictionary = None, - n_frames_per_step: int = 1, - multitask: Optional[Dict] = None, - ) -> SpeechToSpeechDataset: - datasets = [] - for split in splits.split(","): - samples = SpeechToTextDatasetCreator._load_samples_from_tsv(root, split) - ds = cls._from_list( - split, - is_train_split, - samples, - data_cfg, - target_is_code, - target_dictionary, - n_frames_per_step, - multitask, - ) - datasets.append(ds) - return ConcatDataset(datasets) if len(datasets) > 1 else datasets[0] diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/speech_to_text_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/speech_to_text_dataset.py deleted file mode 100644 index d9bfe235a6df868f5ced0f798ee1479a1cee6cad..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/speech_to_text_dataset.py +++ /dev/null @@ -1,560 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import csv -import io -import logging -import re -from collections import defaultdict -from dataclasses import dataclass -from pathlib import Path -from typing import Dict, List, Optional - -import numpy as np -import torch -import torch.nn.functional as F -from fairseq.data import ConcatDataset, Dictionary, FairseqDataset, ResamplingDataset -from fairseq.data import data_utils as fairseq_data_utils -from fairseq.data.audio.audio_utils import ( - FEATURE_OR_SF_AUDIO_FILE_EXTENSIONS, - get_fbank, - get_waveform, - is_npy_data, - is_sf_audio_data, - parse_path, - read_from_stored_zip, -) -from fairseq.data.audio.data_cfg import S2TDataConfig -from fairseq.data.audio.feature_transforms import CompositeAudioFeatureTransform - -logger = logging.getLogger(__name__) - - -def get_features_from_npy_or_audio(path): - ext = Path(path).suffix - if ext not in FEATURE_OR_SF_AUDIO_FILE_EXTENSIONS: - raise ValueError(f'Unsupported file format for "{path}"') - return np.load(path) if ext == ".npy" else get_fbank(path) - - -def get_features_or_waveform_from_stored_zip( - path, - byte_offset, - byte_size, - need_waveform=False, - use_sample_rate=None, -): - assert path.endswith(".zip") - data = read_from_stored_zip(path, byte_offset, byte_size) - f = io.BytesIO(data) - if is_npy_data(data): - features_or_waveform = np.load(f) - elif is_sf_audio_data(data): - features_or_waveform = ( - get_waveform(f, always_2d=False, output_sample_rate=use_sample_rate)[0] - if need_waveform - else get_fbank(f) - ) - else: - raise ValueError(f'Unknown file format for "{path}"') - return features_or_waveform - - -def get_features_or_waveform(path: str, need_waveform=False, use_sample_rate=None): - """Get speech features from .npy file or waveform from .wav/.flac file. - The file may be inside an uncompressed ZIP file and is accessed via byte - offset and length. - - Args: - path (str): File path in the format of "<.npy/.wav/.flac path>" or - "<zip path>:<byte offset>:<byte length>". - need_waveform (bool): return waveform instead of features. - use_sample_rate (int): change sample rate for the input wave file - - Returns: - features_or_waveform (numpy.ndarray): speech features or waveform. - """ - _path, slice_ptr = parse_path(path) - if len(slice_ptr) == 0: - if need_waveform: - return get_waveform( - _path, always_2d=False, output_sample_rate=use_sample_rate - )[0] - return get_features_from_npy_or_audio(_path) - elif len(slice_ptr) == 2: - features_or_waveform = get_features_or_waveform_from_stored_zip( - _path, - slice_ptr[0], - slice_ptr[1], - need_waveform=need_waveform, - use_sample_rate=use_sample_rate, - ) - else: - raise ValueError(f"Invalid path: {path}") - - return features_or_waveform - - -def _collate_frames( - frames: List[torch.Tensor], is_audio_input: bool = False -) -> torch.Tensor: - """ - Convert a list of 2D frames into a padded 3D tensor - Args: - frames (list): list of 2D frames of size L[i]*f_dim. Where L[i] is - length of i-th frame and f_dim is static dimension of features - Returns: - 3D tensor of size len(frames)*len_max*f_dim where len_max is max of L[i] - """ - max_len = max(frame.size(0) for frame in frames) - if is_audio_input: - out = frames[0].new_zeros((len(frames), max_len)) - else: - out = frames[0].new_zeros((len(frames), max_len, frames[0].size(1))) - for i, v in enumerate(frames): - out[i, : v.size(0)] = v - return out - - -@dataclass -class SpeechToTextDatasetItem(object): - index: int - source: torch.Tensor - target: Optional[torch.Tensor] = None - speaker_id: Optional[int] = None - - -class SpeechToTextDataset(FairseqDataset): - LANG_TAG_TEMPLATE = "<lang:{}>" - - def __init__( - self, - split: str, - is_train_split: bool, - cfg: S2TDataConfig, - audio_paths: List[str], - n_frames: List[int], - src_texts: Optional[List[str]] = None, - tgt_texts: Optional[List[str]] = None, - speakers: Optional[List[str]] = None, - src_langs: Optional[List[str]] = None, - tgt_langs: Optional[List[str]] = None, - ids: Optional[List[str]] = None, - tgt_dict: Optional[Dictionary] = None, - pre_tokenizer=None, - bpe_tokenizer=None, - n_frames_per_step=1, - speaker_to_id=None, - append_eos=True, - ): - self.split, self.is_train_split = split, is_train_split - self.cfg = cfg - self.audio_paths, self.n_frames = audio_paths, n_frames - self.n_samples = len(audio_paths) - assert len(n_frames) == self.n_samples > 0 - assert src_texts is None or len(src_texts) == self.n_samples - assert tgt_texts is None or len(tgt_texts) == self.n_samples - assert speakers is None or len(speakers) == self.n_samples - assert src_langs is None or len(src_langs) == self.n_samples - assert tgt_langs is None or len(tgt_langs) == self.n_samples - assert ids is None or len(ids) == self.n_samples - assert (tgt_dict is None and tgt_texts is None) or ( - tgt_dict is not None and tgt_texts is not None - ) - self.src_texts, self.tgt_texts = src_texts, tgt_texts - self.src_langs, self.tgt_langs = src_langs, tgt_langs - self.speakers = speakers - self.tgt_dict = tgt_dict - self.check_tgt_lang_tag() - self.ids = ids - self.shuffle = cfg.shuffle if is_train_split else False - - self.feature_transforms = CompositeAudioFeatureTransform.from_config_dict( - self.cfg.get_feature_transforms(split, is_train_split) - ) - - self.pre_tokenizer = pre_tokenizer - self.bpe_tokenizer = bpe_tokenizer - self.n_frames_per_step = n_frames_per_step - self.speaker_to_id = speaker_to_id - - self.tgt_lens = self.get_tgt_lens_and_check_oov() - self.append_eos = append_eos - - logger.info(self.__repr__()) - - def get_tgt_lens_and_check_oov(self): - if self.tgt_texts is None: - return [0 for _ in range(self.n_samples)] - tgt_lens = [] - n_tokens, n_oov_tokens = 0, 0 - for i in range(self.n_samples): - tokenized = self.get_tokenized_tgt_text(i).split(" ") - oov_tokens = [ - t - for t in tokenized - if self.tgt_dict.index(t) == self.tgt_dict.unk_index - ] - n_tokens += len(tokenized) - n_oov_tokens += len(oov_tokens) - tgt_lens.append(len(tokenized)) - logger.info(f"'{self.split}' has {n_oov_tokens / n_tokens * 100:.2f}% OOV") - return tgt_lens - - def __repr__(self): - return ( - self.__class__.__name__ - + f'(split="{self.split}", n_samples={self.n_samples:_}, ' - f"prepend_tgt_lang_tag={self.cfg.prepend_tgt_lang_tag}, " - f"shuffle={self.shuffle}, transforms={self.feature_transforms}, " - f"n_frames_per_step={self.n_frames_per_step}" - ) - - @classmethod - def is_lang_tag(cls, token): - pattern = cls.LANG_TAG_TEMPLATE.replace("{}", "(.*)") - return re.match(pattern, token) - - def check_tgt_lang_tag(self): - if self.cfg.prepend_tgt_lang_tag: - assert self.tgt_langs is not None and self.tgt_dict is not None - tgt_lang_tags = [ - self.LANG_TAG_TEMPLATE.format(t) for t in set(self.tgt_langs) - ] - assert all(t in self.tgt_dict for t in tgt_lang_tags) - - @classmethod - def tokenize(cls, tokenizer, text: str): - return text if tokenizer is None else tokenizer.encode(text) - - def get_tokenized_tgt_text(self, index: int): - text = self.tokenize(self.pre_tokenizer, self.tgt_texts[index]) - text = self.tokenize(self.bpe_tokenizer, text) - return text - - def pack_frames(self, feature: torch.Tensor): - if self.n_frames_per_step == 1: - return feature - n_packed_frames = feature.shape[0] // self.n_frames_per_step - feature = feature[: self.n_frames_per_step * n_packed_frames] - return feature.reshape(n_packed_frames, -1) - - @classmethod - def get_lang_tag_idx(cls, lang: str, dictionary: Dictionary): - lang_tag_idx = dictionary.index(cls.LANG_TAG_TEMPLATE.format(lang)) - assert lang_tag_idx != dictionary.unk() - return lang_tag_idx - - def _get_source_audio(self, index: int) -> torch.Tensor: - source = get_features_or_waveform( - self.audio_paths[index], - need_waveform=self.cfg.use_audio_input, - use_sample_rate=self.cfg.use_sample_rate, - ) - if self.cfg.use_audio_input: - source = torch.from_numpy(source).float() - if self.cfg.standardize_audio: - with torch.no_grad(): - source = F.layer_norm(source, source.shape) - else: - if self.feature_transforms is not None: - source = self.feature_transforms(source) - source = torch.from_numpy(source).float() - return source - - def __getitem__(self, index: int) -> SpeechToTextDatasetItem: - source = self._get_source_audio(index) - source = self.pack_frames(source) - - target = None - if self.tgt_texts is not None: - tokenized = self.get_tokenized_tgt_text(index) - target = self.tgt_dict.encode_line( - tokenized, add_if_not_exist=False, append_eos=self.append_eos - ).long() - if self.cfg.prepend_tgt_lang_tag: - lang_tag_idx = self.get_lang_tag_idx( - self.tgt_langs[index], self.tgt_dict - ) - target = torch.cat((torch.LongTensor([lang_tag_idx]), target), 0) - - if self.cfg.prepend_bos_and_append_tgt_lang_tag: - bos = torch.LongTensor([self.tgt_dict.bos()]) - lang_tag_idx = self.get_lang_tag_idx(self.tgt_langs[index], self.tgt_dict) - assert lang_tag_idx != self.tgt_dict.unk() - lang_tag_idx = torch.LongTensor([lang_tag_idx]) - target = torch.cat((bos, target, lang_tag_idx), 0) - - speaker_id = None - if self.speaker_to_id is not None: - speaker_id = self.speaker_to_id[self.speakers[index]] - return SpeechToTextDatasetItem( - index=index, source=source, target=target, speaker_id=speaker_id - ) - - def __len__(self): - return self.n_samples - - def collater( - self, samples: List[SpeechToTextDatasetItem], return_order: bool = False - ) -> Dict: - if len(samples) == 0: - return {} - indices = torch.tensor([x.index for x in samples], dtype=torch.long) - frames = _collate_frames([x.source for x in samples], self.cfg.use_audio_input) - # sort samples by descending number of frames - n_frames = torch.tensor([x.source.size(0) for x in samples], dtype=torch.long) - n_frames, order = n_frames.sort(descending=True) - indices = indices.index_select(0, order) - frames = frames.index_select(0, order) - - target, target_lengths = None, None - prev_output_tokens = None - ntokens = None - if self.tgt_texts is not None: - target = fairseq_data_utils.collate_tokens( - [x.target for x in samples], - self.tgt_dict.pad(), - self.tgt_dict.eos(), - left_pad=False, - move_eos_to_beginning=False, - ) - target = target.index_select(0, order) - target_lengths = torch.tensor( - [x.target.size(0) for x in samples], dtype=torch.long - ).index_select(0, order) - prev_output_tokens = fairseq_data_utils.collate_tokens( - [x.target for x in samples], - self.tgt_dict.pad(), - eos_idx=None, - left_pad=False, - move_eos_to_beginning=True, - ) - prev_output_tokens = prev_output_tokens.index_select(0, order) - ntokens = sum(x.target.size(0) for x in samples) - - speaker = None - if self.speaker_to_id is not None: - speaker = ( - torch.tensor([s.speaker_id for s in samples], dtype=torch.long) - .index_select(0, order) - .view(-1, 1) - ) - - net_input = { - "src_tokens": frames, - "src_lengths": n_frames, - "prev_output_tokens": prev_output_tokens, - } - out = { - "id": indices, - "net_input": net_input, - "speaker": speaker, - "target": target, - "target_lengths": target_lengths, - "ntokens": ntokens, - "nsentences": len(samples), - } - if return_order: - out["order"] = order - return out - - def num_tokens(self, index): - return self.n_frames[index] - - def size(self, index): - return self.n_frames[index], self.tgt_lens[index] - - @property - def sizes(self): - return np.array(self.n_frames) - - @property - def can_reuse_epoch_itr_across_epochs(self): - return True - - def ordered_indices(self): - if self.shuffle: - order = [np.random.permutation(len(self))] - else: - order = [np.arange(len(self))] - # first by descending order of # of frames then by original/random order - order.append([-n for n in self.n_frames]) - return np.lexsort(order) - - def prefetch(self, indices): - raise False - - -class SpeechToTextDatasetCreator(object): - # mandatory columns - KEY_ID, KEY_AUDIO, KEY_N_FRAMES = "id", "audio", "n_frames" - KEY_TGT_TEXT = "tgt_text" - # optional columns - KEY_SPEAKER, KEY_SRC_TEXT = "speaker", "src_text" - KEY_SRC_LANG, KEY_TGT_LANG = "src_lang", "tgt_lang" - # default values - DEFAULT_SPEAKER = DEFAULT_SRC_TEXT = DEFAULT_LANG = "" - - @classmethod - def _from_list( - cls, - split_name: str, - is_train_split, - samples: List[Dict], - cfg: S2TDataConfig, - tgt_dict, - pre_tokenizer, - bpe_tokenizer, - n_frames_per_step, - speaker_to_id, - ) -> SpeechToTextDataset: - audio_root = Path(cfg.audio_root) - ids = [s[cls.KEY_ID] for s in samples] - audio_paths = [(audio_root / s[cls.KEY_AUDIO]).as_posix() for s in samples] - n_frames = [int(s[cls.KEY_N_FRAMES]) for s in samples] - tgt_texts = [s[cls.KEY_TGT_TEXT] for s in samples] - src_texts = [s.get(cls.KEY_SRC_TEXT, cls.DEFAULT_SRC_TEXT) for s in samples] - speakers = [s.get(cls.KEY_SPEAKER, cls.DEFAULT_SPEAKER) for s in samples] - src_langs = [s.get(cls.KEY_SRC_LANG, cls.DEFAULT_LANG) for s in samples] - tgt_langs = [s.get(cls.KEY_TGT_LANG, cls.DEFAULT_LANG) for s in samples] - return SpeechToTextDataset( - split_name, - is_train_split, - cfg, - audio_paths, - n_frames, - src_texts=src_texts, - tgt_texts=tgt_texts, - speakers=speakers, - src_langs=src_langs, - tgt_langs=tgt_langs, - ids=ids, - tgt_dict=tgt_dict, - pre_tokenizer=pre_tokenizer, - bpe_tokenizer=bpe_tokenizer, - n_frames_per_step=n_frames_per_step, - speaker_to_id=speaker_to_id, - ) - - @classmethod - def get_size_ratios( - cls, datasets: List[SpeechToTextDataset], alpha: float = 1.0 - ) -> List[float]: - """Size ratios for temperature-based sampling - (https://arxiv.org/abs/1907.05019)""" - - id_to_lp, lp_to_sz = {}, defaultdict(int) - for ds in datasets: - lang_pairs = {f"{s}->{t}" for s, t in zip(ds.src_langs, ds.tgt_langs)} - assert len(lang_pairs) == 1 - lang_pair = list(lang_pairs)[0] - id_to_lp[ds.split] = lang_pair - lp_to_sz[lang_pair] += sum(ds.n_frames) - - sz_sum = sum(v for v in lp_to_sz.values()) - lp_to_prob = {k: v / sz_sum for k, v in lp_to_sz.items()} - lp_to_tgt_prob = {k: v**alpha for k, v in lp_to_prob.items()} - prob_sum = sum(v for v in lp_to_tgt_prob.values()) - lp_to_tgt_prob = {k: v / prob_sum for k, v in lp_to_tgt_prob.items()} - lp_to_sz_ratio = { - k: (lp_to_tgt_prob[k] * sz_sum) / v for k, v in lp_to_sz.items() - } - size_ratio = [lp_to_sz_ratio[id_to_lp[ds.split]] for ds in datasets] - - p_formatted = { - k: f"{lp_to_prob[k]:.3f}->{lp_to_tgt_prob[k]:.3f}" for k in lp_to_sz - } - logger.info(f"sampling probability balancing: {p_formatted}") - sr_formatted = {ds.split: f"{r:.3f}" for ds, r in zip(datasets, size_ratio)} - logger.info(f"balanced sampling size ratio: {sr_formatted}") - return size_ratio - - @classmethod - def _load_samples_from_tsv(cls, root: str, split: str): - tsv_path = Path(root) / f"{split}.tsv" - if not tsv_path.is_file(): - raise FileNotFoundError(f"Dataset not found: {tsv_path}") - with open(tsv_path) as f: - reader = csv.DictReader( - f, - delimiter="\t", - quotechar=None, - doublequote=False, - lineterminator="\n", - quoting=csv.QUOTE_NONE, - ) - samples = [dict(e) for e in reader] - if len(samples) == 0: - raise ValueError(f"Empty manifest: {tsv_path}") - return samples - - @classmethod - def _from_tsv( - cls, - root: str, - cfg: S2TDataConfig, - split: str, - tgt_dict, - is_train_split: bool, - pre_tokenizer, - bpe_tokenizer, - n_frames_per_step, - speaker_to_id, - ) -> SpeechToTextDataset: - samples = cls._load_samples_from_tsv(root, split) - return cls._from_list( - split, - is_train_split, - samples, - cfg, - tgt_dict, - pre_tokenizer, - bpe_tokenizer, - n_frames_per_step, - speaker_to_id, - ) - - @classmethod - def from_tsv( - cls, - root: str, - cfg: S2TDataConfig, - splits: str, - tgt_dict, - pre_tokenizer, - bpe_tokenizer, - is_train_split: bool, - epoch: int, - seed: int, - n_frames_per_step: int = 1, - speaker_to_id=None, - ) -> SpeechToTextDataset: - datasets = [ - cls._from_tsv( - root, - cfg, - split, - tgt_dict, - is_train_split, - pre_tokenizer, - bpe_tokenizer, - n_frames_per_step, - speaker_to_id, - ) - for split in splits.split(",") - ] - - if is_train_split and len(datasets) > 1 and cfg.sampling_alpha != 1.0: - # temperature-based sampling - size_ratios = cls.get_size_ratios(datasets, alpha=cfg.sampling_alpha) - datasets = [ - ResamplingDataset( - d, size_ratio=r, seed=seed, epoch=epoch, replace=(r >= 1.0) - ) - for r, d in zip(size_ratios, datasets) - ] - - return ConcatDataset(datasets) if len(datasets) > 1 else datasets[0] diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/speech_to_text_joint_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/speech_to_text_joint_dataset.py deleted file mode 100644 index aee30d0df5aeabc54bcc1401bd5aa5c9d184d5f4..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/speech_to_text_joint_dataset.py +++ /dev/null @@ -1,358 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import logging -from pathlib import Path -from typing import Dict, List, NamedTuple, Optional - -import torch -from fairseq.data import ConcatDataset, Dictionary, ResamplingDataset -from fairseq.data import data_utils as fairseq_data_utils -from fairseq.data.audio.speech_to_text_dataset import ( - S2TDataConfig, - SpeechToTextDataset, - SpeechToTextDatasetCreator, -) - -logger = logging.getLogger(__name__) - - -class S2TJointDataConfig(S2TDataConfig): - """Wrapper class for data config YAML""" - - @property - def src_vocab_filename(self): - """fairseq vocabulary file under data root""" - return self.config.get("src_vocab_filename", "src_dict.txt") - - @property - def src_pre_tokenizer(self) -> Dict: - """Pre-tokenizer to apply before subword tokenization. Returning - a dictionary with `tokenizer` providing the tokenizer name and - the other items providing the tokenizer-specific arguments. - Tokenizers are defined in `fairseq.data.encoders.*`""" - return self.config.get("src_pre_tokenizer", {"tokenizer": None}) - - @property - def src_bpe_tokenizer(self) -> Dict: - """Subword tokenizer to apply on source text after pre-tokenization. - Returning a dictionary with `bpe` providing the tokenizer name and - the other items providing the tokenizer-specific arguments. - Tokenizers are defined in `fairseq.data.encoders.*`""" - return self.config.get("src_bpe_tokenizer", {"bpe": None}) - - @property - def prepend_tgt_lang_tag_no_change(self) -> bool: - """Prepend target lang ID token as the prev_output_tokens BOS (e.g. for - to-many multilingual setting). No change needed during inference. - This option is deprecated and replaced by prepend_tgt_lang_tag_as_bos. - """ - value = self.config.get("prepend_tgt_lang_tag_no_change", None) - if value is None: - return self.config.get("prepend_tgt_lang_tag_as_bos", False) - return value - - @property - def sampling_text_alpha(self): - """Hyper-parameter alpha = 1/T for temperature-based resampling. (text - input only) (alpha = 1 for no resampling)""" - return self.config.get("sampling_text_alpha", 1.0) - - -class SpeechToTextJointDatasetItem(NamedTuple): - index: int - source: torch.Tensor - target: Optional[torch.Tensor] = None - src_txt_tokens: Optional[torch.Tensor] = None - tgt_lang_tag: Optional[int] = None - src_lang_tag: Optional[int] = None - tgt_alignment: Optional[torch.Tensor] = None - - -# use_src_lang_id: -# 0: don't use src_lang_id -# 1: attach src_lang_id to the src_txt_tokens as eos -class SpeechToTextJointDataset(SpeechToTextDataset): - def __init__( - self, - split: str, - is_train_split: bool, - cfg: S2TJointDataConfig, - audio_paths: List[str], - n_frames: List[int], - src_texts: Optional[List[str]] = None, - tgt_texts: Optional[List[str]] = None, - speakers: Optional[List[str]] = None, - src_langs: Optional[List[str]] = None, - tgt_langs: Optional[List[str]] = None, - ids: Optional[List[str]] = None, - tgt_dict: Optional[Dictionary] = None, - src_dict: Optional[Dictionary] = None, - pre_tokenizer=None, - bpe_tokenizer=None, - src_pre_tokenizer=None, - src_bpe_tokenizer=None, - append_eos: Optional[bool] = True, - alignment: Optional[List[str]] = None, - use_src_lang_id: Optional[int] = 0, - ): - super().__init__( - split, - is_train_split, - cfg, - audio_paths, - n_frames, - src_texts=src_texts, - tgt_texts=tgt_texts, - speakers=speakers, - src_langs=src_langs, - tgt_langs=tgt_langs, - ids=ids, - tgt_dict=tgt_dict, - pre_tokenizer=pre_tokenizer, - bpe_tokenizer=bpe_tokenizer, - append_eos=append_eos, - ) - - self.src_dict = src_dict - self.src_pre_tokenizer = src_pre_tokenizer - self.src_bpe_tokenizer = src_bpe_tokenizer - self.alignment = None - self.use_src_lang_id = use_src_lang_id - if alignment is not None: - self.alignment = [ - [float(s) for s in sample.split()] for sample in alignment - ] - - def get_tokenized_src_text(self, index: int): - text = self.tokenize(self.src_pre_tokenizer, self.src_texts[index]) - text = self.tokenize(self.src_bpe_tokenizer, text) - return text - - def __getitem__(self, index: int) -> SpeechToTextJointDatasetItem: - s2t_dataset_item = super().__getitem__(index) - src_tokens = None - src_lang_tag = None - if self.src_texts is not None and self.src_dict is not None: - src_tokens = self.get_tokenized_src_text(index) - src_tokens = self.src_dict.encode_line( - src_tokens, add_if_not_exist=False, append_eos=True - ).long() - if self.use_src_lang_id > 0: - src_lang_tag = self.get_lang_tag_idx( - self.src_langs[index], self.src_dict - ) - tgt_lang_tag = None - if self.cfg.prepend_tgt_lang_tag_no_change: - # prepend_tgt_lang_tag_no_change: modify prev_output_tokens instead - tgt_lang_tag = self.get_lang_tag_idx(self.tgt_langs[index], self.tgt_dict) - ali = None - if self.alignment is not None: - ali = torch.Tensor(self.alignment[index]).float() - - return SpeechToTextJointDatasetItem( - index=index, - source=s2t_dataset_item.source, - target=s2t_dataset_item.target, - src_txt_tokens=src_tokens, - tgt_lang_tag=tgt_lang_tag, - src_lang_tag=src_lang_tag, - tgt_alignment=ali, - ) - - def __len__(self): - return self.n_samples - - def collater(self, samples: List[SpeechToTextJointDatasetItem]) -> Dict: - s2t_out = super().collater(samples, return_order=True) - if s2t_out == {}: - return s2t_out - net_input, order = s2t_out["net_input"], s2t_out["order"] - - if self.src_texts is not None and self.src_dict is not None: - src_txt_tokens = fairseq_data_utils.collate_tokens( - [x.src_txt_tokens for x in samples], - self.src_dict.pad(), - self.src_dict.eos(), - left_pad=False, - move_eos_to_beginning=False, - ) - src_txt_lengths = torch.tensor( - [x.src_txt_tokens.size()[0] for x in samples], dtype=torch.long - ) - if self.use_src_lang_id > 0: - src_lang_idxs = torch.tensor( - [s.src_lang_tag for s in samples], dtype=src_txt_tokens.dtype - ) - if self.use_src_lang_id == 1: # replace eos with lang_id - eos_idx = src_txt_lengths - 1 - src_txt_tokens.scatter_( - 1, eos_idx.view(-1, 1), src_lang_idxs.view(-1, 1) - ) - else: - raise NotImplementedError("Implementation is required") - - src_txt_tokens = src_txt_tokens.index_select(0, order) - src_txt_lengths = src_txt_lengths.index_select(0, order) - net_input["src_txt_tokens"] = src_txt_tokens - net_input["src_txt_lengths"] = src_txt_lengths - - net_input["alignment"] = None - if self.alignment is not None: - max_len = max([s.tgt_alignment.size(0) for s in samples]) - alignment = torch.ones(len(samples), max_len).float() - for i, s in enumerate(samples): - cur_len = s.tgt_alignment.size(0) - alignment[i][:cur_len].copy_(s.tgt_alignment) - net_input["alignment"] = alignment.index_select(0, order) - - if self.tgt_texts is not None and samples[0].tgt_lang_tag is not None: - for i in range(len(samples)): - net_input["prev_output_tokens"][i][0] = samples[order[i]].tgt_lang_tag - - out = { - "id": s2t_out["id"], - "net_input": net_input, - "target": s2t_out["target"], - "target_lengths": s2t_out["target_lengths"], - "ntokens": s2t_out["ntokens"], - "nsentences": len(samples), - } - return out - - -class SpeechToTextJointDatasetCreator(SpeechToTextDatasetCreator): - KEY_ALIGN = "align" - - @classmethod - def _from_list( - cls, - split_name: str, - is_train_split, - samples: List[Dict], - cfg: S2TJointDataConfig, - tgt_dict, - src_dict, - pre_tokenizer, - bpe_tokenizer, - src_pre_tokenizer, - src_bpe_tokenizer, - append_eos, - use_src_lang_id, - ) -> SpeechToTextJointDataset: - audio_root = Path(cfg.audio_root) - ids = [s[cls.KEY_ID] for s in samples] - audio_paths = [(audio_root / s[cls.KEY_AUDIO]).as_posix() for s in samples] - n_frames = [int(s[cls.KEY_N_FRAMES]) for s in samples] - tgt_texts = [s[cls.KEY_TGT_TEXT] for s in samples] - src_texts = [s.get(cls.KEY_SRC_TEXT, cls.DEFAULT_SRC_TEXT) for s in samples] - speakers = [s.get(cls.KEY_SPEAKER, cls.DEFAULT_SPEAKER) for s in samples] - src_langs = [s.get(cls.KEY_SRC_LANG, cls.DEFAULT_LANG) for s in samples] - tgt_langs = [s.get(cls.KEY_TGT_LANG, cls.DEFAULT_LANG) for s in samples] - tgt_alignment = None - if cls.KEY_ALIGN in samples[0].keys(): - tgt_alignment = [s[cls.KEY_ALIGN] for s in samples] - return SpeechToTextJointDataset( - split_name, - is_train_split, - cfg, - audio_paths, - n_frames, - src_texts=src_texts, - tgt_texts=tgt_texts, - speakers=speakers, - src_langs=src_langs, - tgt_langs=tgt_langs, - ids=ids, - tgt_dict=tgt_dict, - src_dict=src_dict, - pre_tokenizer=pre_tokenizer, - bpe_tokenizer=bpe_tokenizer, - src_pre_tokenizer=src_pre_tokenizer, - src_bpe_tokenizer=src_bpe_tokenizer, - append_eos=append_eos, - alignment=tgt_alignment, - use_src_lang_id=use_src_lang_id, - ) - - @classmethod - def _from_tsv( - cls, - root: str, - cfg: S2TJointDataConfig, - split: str, - tgt_dict, - src_dict, - is_train_split: bool, - pre_tokenizer, - bpe_tokenizer, - src_pre_tokenizer, - src_bpe_tokenizer, - append_eos: bool, - use_src_lang_id: int, - ) -> SpeechToTextJointDataset: - samples = cls._load_samples_from_tsv(root, split) - return cls._from_list( - split, - is_train_split, - samples, - cfg, - tgt_dict, - src_dict, - pre_tokenizer, - bpe_tokenizer, - src_pre_tokenizer, - src_bpe_tokenizer, - append_eos, - use_src_lang_id, - ) - - @classmethod - def from_tsv( - cls, - root: str, - cfg: S2TJointDataConfig, - splits: str, - tgt_dict, - src_dict, - pre_tokenizer, - bpe_tokenizer, - src_pre_tokenizer, - src_bpe_tokenizer, - is_train_split: bool, - epoch: int, - seed: int, - append_eos: Optional[bool] = True, - use_src_lang_id: Optional[int] = 0, - ) -> SpeechToTextJointDataset: - datasets = [ - cls._from_tsv( - root, - cfg, - split, - tgt_dict, - src_dict, - is_train_split, - pre_tokenizer, - bpe_tokenizer, - src_pre_tokenizer, - src_bpe_tokenizer, - append_eos=append_eos, - use_src_lang_id=use_src_lang_id, - ) - for split in splits.split(",") - ] - - if is_train_split and len(datasets) > 1 and cfg.sampling_alpha != 1.0: - # temperature-based sampling - size_ratios = cls.get_size_ratios(datasets, alpha=cfg.sampling_alpha) - datasets = [ - ResamplingDataset( - d, size_ratio=r, seed=seed, epoch=epoch, replace=(r >= 1.0) - ) - for r, d in zip(size_ratios, datasets) - ] - - return ConcatDataset(datasets) if len(datasets) > 1 else datasets[0] diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/text_to_speech_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/text_to_speech_dataset.py deleted file mode 100644 index a42ce3545b1e0ce9dca05e6c352fc9a3aac7e1b5..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/audio/text_to_speech_dataset.py +++ /dev/null @@ -1,248 +0,0 @@ -# Copyright (c) 2017-present, Facebook, Inc. -# All rights reserved. -# -# This source code is licensed under the license found in the LICENSE file in -# the root directory of this source tree. An additional grant of patent rights -# can be found in the PATENTS file in the same directory.abs - -from dataclasses import dataclass -from pathlib import Path -from typing import Any, Dict, List, Optional - -import numpy as np -import torch -from fairseq.data import Dictionary -from fairseq.data import data_utils as fairseq_data_utils -from fairseq.data.audio.speech_to_text_dataset import ( - S2TDataConfig, - SpeechToTextDataset, - SpeechToTextDatasetCreator, - _collate_frames, - get_features_or_waveform, -) - - -@dataclass -class TextToSpeechDatasetItem(object): - index: int - source: torch.Tensor - target: Optional[torch.Tensor] = None - speaker_id: Optional[int] = None - duration: Optional[torch.Tensor] = None - pitch: Optional[torch.Tensor] = None - energy: Optional[torch.Tensor] = None - - -class TextToSpeechDataset(SpeechToTextDataset): - def __init__( - self, - split: str, - is_train_split: bool, - cfg: S2TDataConfig, - audio_paths: List[str], - n_frames: List[int], - src_texts: Optional[List[str]] = None, - tgt_texts: Optional[List[str]] = None, - speakers: Optional[List[str]] = None, - src_langs: Optional[List[str]] = None, - tgt_langs: Optional[List[str]] = None, - ids: Optional[List[str]] = None, - tgt_dict: Optional[Dictionary] = None, - pre_tokenizer=None, - bpe_tokenizer=None, - n_frames_per_step=1, - speaker_to_id=None, - durations: Optional[List[List[int]]] = None, - pitches: Optional[List[str]] = None, - energies: Optional[List[str]] = None, - ): - super(TextToSpeechDataset, self).__init__( - split, - is_train_split, - cfg, - audio_paths, - n_frames, - src_texts=src_texts, - tgt_texts=tgt_texts, - speakers=speakers, - src_langs=src_langs, - tgt_langs=tgt_langs, - ids=ids, - tgt_dict=tgt_dict, - pre_tokenizer=pre_tokenizer, - bpe_tokenizer=bpe_tokenizer, - n_frames_per_step=n_frames_per_step, - speaker_to_id=speaker_to_id, - ) - self.durations = durations - self.pitches = pitches - self.energies = energies - - def __getitem__(self, index: int) -> TextToSpeechDatasetItem: - s2t_item = super().__getitem__(index) - - duration, pitch, energy = None, None, None - if self.durations is not None: - duration = torch.tensor( - self.durations[index] + [0], dtype=torch.long # pad 0 for EOS - ) - if self.pitches is not None: - pitch = get_features_or_waveform(self.pitches[index]) - pitch = torch.from_numpy( - np.concatenate((pitch, [0])) # pad 0 for EOS - ).float() - if self.energies is not None: - energy = get_features_or_waveform(self.energies[index]) - energy = torch.from_numpy( - np.concatenate((energy, [0])) # pad 0 for EOS - ).float() - return TextToSpeechDatasetItem( - index=index, - source=s2t_item.source, - target=s2t_item.target, - speaker_id=s2t_item.speaker_id, - duration=duration, - pitch=pitch, - energy=energy, - ) - - def collater(self, samples: List[TextToSpeechDatasetItem]) -> Dict[str, Any]: - if len(samples) == 0: - return {} - - src_lengths, order = torch.tensor( - [s.target.shape[0] for s in samples], dtype=torch.long - ).sort(descending=True) - id_ = torch.tensor([s.index for s in samples], dtype=torch.long).index_select( - 0, order - ) - feat = _collate_frames( - [s.source for s in samples], self.cfg.use_audio_input - ).index_select(0, order) - target_lengths = torch.tensor( - [s.source.shape[0] for s in samples], dtype=torch.long - ).index_select(0, order) - - src_tokens = fairseq_data_utils.collate_tokens( - [s.target for s in samples], - self.tgt_dict.pad(), - self.tgt_dict.eos(), - left_pad=False, - move_eos_to_beginning=False, - ).index_select(0, order) - - speaker = None - if self.speaker_to_id is not None: - speaker = ( - torch.tensor([s.speaker_id for s in samples], dtype=torch.long) - .index_select(0, order) - .view(-1, 1) - ) - - bsz, _, d = feat.size() - prev_output_tokens = torch.cat( - (feat.new_zeros((bsz, 1, d)), feat[:, :-1, :]), dim=1 - ) - - durations, pitches, energies = None, None, None - if self.durations is not None: - durations = fairseq_data_utils.collate_tokens( - [s.duration for s in samples], 0 - ).index_select(0, order) - assert src_tokens.shape[1] == durations.shape[1] - if self.pitches is not None: - pitches = _collate_frames([s.pitch for s in samples], True) - pitches = pitches.index_select(0, order) - assert src_tokens.shape[1] == pitches.shape[1] - if self.energies is not None: - energies = _collate_frames([s.energy for s in samples], True) - energies = energies.index_select(0, order) - assert src_tokens.shape[1] == energies.shape[1] - src_texts = [self.tgt_dict.string(samples[i].target) for i in order] - - return { - "id": id_, - "net_input": { - "src_tokens": src_tokens, - "src_lengths": src_lengths, - "prev_output_tokens": prev_output_tokens, - }, - "speaker": speaker, - "target": feat, - "durations": durations, - "pitches": pitches, - "energies": energies, - "target_lengths": target_lengths, - "ntokens": sum(target_lengths).item(), - "nsentences": len(samples), - "src_texts": src_texts, - } - - -class TextToSpeechDatasetCreator(SpeechToTextDatasetCreator): - KEY_DURATION = "duration" - KEY_PITCH = "pitch" - KEY_ENERGY = "energy" - - @classmethod - def _from_list( - cls, - split_name: str, - is_train_split, - samples: List[Dict], - cfg: S2TDataConfig, - tgt_dict, - pre_tokenizer, - bpe_tokenizer, - n_frames_per_step, - speaker_to_id, - ) -> TextToSpeechDataset: - audio_root = Path(cfg.audio_root) - ids = [s[cls.KEY_ID] for s in samples] - audio_paths = [(audio_root / s[cls.KEY_AUDIO]).as_posix() for s in samples] - n_frames = [int(s[cls.KEY_N_FRAMES]) for s in samples] - tgt_texts = [s[cls.KEY_TGT_TEXT] for s in samples] - src_texts = [s.get(cls.KEY_SRC_TEXT, cls.DEFAULT_SRC_TEXT) for s in samples] - speakers = [s.get(cls.KEY_SPEAKER, cls.DEFAULT_SPEAKER) for s in samples] - src_langs = [s.get(cls.KEY_SRC_LANG, cls.DEFAULT_LANG) for s in samples] - tgt_langs = [s.get(cls.KEY_TGT_LANG, cls.DEFAULT_LANG) for s in samples] - - durations = [s.get(cls.KEY_DURATION, None) for s in samples] - durations = [ - None if dd is None else [int(d) for d in dd.split(" ")] for dd in durations - ] - durations = None if any(dd is None for dd in durations) else durations - - pitches = [s.get(cls.KEY_PITCH, None) for s in samples] - pitches = [ - None if pp is None else (audio_root / pp).as_posix() for pp in pitches - ] - pitches = None if any(pp is None for pp in pitches) else pitches - - energies = [s.get(cls.KEY_ENERGY, None) for s in samples] - energies = [ - None if ee is None else (audio_root / ee).as_posix() for ee in energies - ] - energies = None if any(ee is None for ee in energies) else energies - - return TextToSpeechDataset( - split_name, - is_train_split, - cfg, - audio_paths, - n_frames, - src_texts, - tgt_texts, - speakers, - src_langs, - tgt_langs, - ids, - tgt_dict, - pre_tokenizer, - bpe_tokenizer, - n_frames_per_step, - speaker_to_id, - durations, - pitches, - energies, - ) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/backtranslation_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/backtranslation_dataset.py deleted file mode 100644 index 8f70c90df3d237077537993e125d366c95292f1a..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/backtranslation_dataset.py +++ /dev/null @@ -1,165 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import torch -from fairseq import utils - -from . import FairseqDataset - - -def backtranslate_samples(samples, collate_fn, generate_fn, cuda=True): - """Backtranslate a list of samples. - - Given an input (*samples*) of the form: - - [{'id': 1, 'source': 'hallo welt'}] - - this will return: - - [{'id': 1, 'source': 'hello world', 'target': 'hallo welt'}] - - Args: - samples (List[dict]): samples to backtranslate. Individual samples are - expected to have a 'source' key, which will become the 'target' - after backtranslation. - collate_fn (callable): function to collate samples into a mini-batch - generate_fn (callable): function to generate backtranslations - cuda (bool): use GPU for generation (default: ``True``) - - Returns: - List[dict]: an updated list of samples with a backtranslated source - """ - collated_samples = collate_fn(samples) - s = utils.move_to_cuda(collated_samples) if cuda else collated_samples - generated_sources = generate_fn(s) - - id_to_src = {sample["id"]: sample["source"] for sample in samples} - - # Go through each tgt sentence in batch and its corresponding best - # generated hypothesis and create a backtranslation data pair - # {id: id, source: generated backtranslation, target: original tgt} - return [ - { - "id": id.item(), - "target": id_to_src[id.item()], - "source": hypos[0]["tokens"].cpu(), - } - for id, hypos in zip(collated_samples["id"], generated_sources) - ] - - -class BacktranslationDataset(FairseqDataset): - """ - Sets up a backtranslation dataset which takes a tgt batch, generates - a src using a tgt-src backtranslation function (*backtranslation_fn*), - and returns the corresponding `{generated src, input tgt}` batch. - - Args: - tgt_dataset (~fairseq.data.FairseqDataset): the dataset to be - backtranslated. Only the source side of this dataset will be used. - After backtranslation, the source sentences in this dataset will be - returned as the targets. - src_dict (~fairseq.data.Dictionary): the dictionary of backtranslated - sentences. - tgt_dict (~fairseq.data.Dictionary, optional): the dictionary of - sentences to be backtranslated. - backtranslation_fn (callable, optional): function to call to generate - backtranslations. This is typically the `generate` method of a - :class:`~fairseq.sequence_generator.SequenceGenerator` object. - Pass in None when it is not available at initialization time, and - use set_backtranslation_fn function to set it when available. - output_collater (callable, optional): function to call on the - backtranslated samples to create the final batch - (default: ``tgt_dataset.collater``). - cuda: use GPU for generation - """ - - def __init__( - self, - tgt_dataset, - src_dict, - tgt_dict=None, - backtranslation_fn=None, - output_collater=None, - cuda=True, - **kwargs - ): - self.tgt_dataset = tgt_dataset - self.backtranslation_fn = backtranslation_fn - self.output_collater = ( - output_collater if output_collater is not None else tgt_dataset.collater - ) - self.cuda = cuda if torch.cuda.is_available() else False - self.src_dict = src_dict - self.tgt_dict = tgt_dict - - def __getitem__(self, index): - """ - Returns a single sample from *tgt_dataset*. Note that backtranslation is - not applied in this step; use :func:`collater` instead to backtranslate - a batch of samples. - """ - return self.tgt_dataset[index] - - def __len__(self): - return len(self.tgt_dataset) - - def set_backtranslation_fn(self, backtranslation_fn): - self.backtranslation_fn = backtranslation_fn - - def collater(self, samples): - """Merge and backtranslate a list of samples to form a mini-batch. - - Using the samples from *tgt_dataset*, load a collated target sample to - feed to the backtranslation model. Then take the backtranslation with - the best score as the source and the original input as the target. - - Note: we expect *tgt_dataset* to provide a function `collater()` that - will collate samples into the format expected by *backtranslation_fn*. - After backtranslation, we will feed the new list of samples (i.e., the - `(backtranslated source, original source)` pairs) to *output_collater* - and return the result. - - Args: - samples (List[dict]): samples to backtranslate and collate - - Returns: - dict: a mini-batch with keys coming from *output_collater* - """ - if samples[0].get("is_dummy", False): - return samples - samples = backtranslate_samples( - samples=samples, - collate_fn=self.tgt_dataset.collater, - generate_fn=(lambda net_input: self.backtranslation_fn(net_input)), - cuda=self.cuda, - ) - return self.output_collater(samples) - - def num_tokens(self, index): - """Just use the tgt dataset num_tokens""" - return self.tgt_dataset.num_tokens(index) - - def ordered_indices(self): - """Just use the tgt dataset ordered_indices""" - return self.tgt_dataset.ordered_indices() - - def size(self, index): - """Return an example's size as a float or tuple. This value is used - when filtering a dataset with ``--max-positions``. - - Note: we use *tgt_dataset* to approximate the length of the source - sentence, since we do not know the actual length until after - backtranslation. - """ - tgt_size = self.tgt_dataset.size(index)[0] - return (tgt_size, tgt_size) - - @property - def supports_prefetch(self): - return getattr(self.tgt_dataset, "supports_prefetch", False) - - def prefetch(self, indices): - return self.tgt_dataset.prefetch(indices) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/base_wrapper_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/base_wrapper_dataset.py deleted file mode 100644 index 134d398b47dc73c8807759188504aee205b3b34d..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/base_wrapper_dataset.py +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from torch.utils.data.dataloader import default_collate - -from . import FairseqDataset - - -class BaseWrapperDataset(FairseqDataset): - def __init__(self, dataset): - super().__init__() - self.dataset = dataset - - def __getitem__(self, index): - return self.dataset[index] - - def __len__(self): - return len(self.dataset) - - def collater(self, samples): - if hasattr(self.dataset, "collater"): - return self.dataset.collater(samples) - else: - return default_collate(samples) - - @property - def sizes(self): - return self.dataset.sizes - - def num_tokens(self, index): - return self.dataset.num_tokens(index) - - def size(self, index): - return self.dataset.size(index) - - def ordered_indices(self): - return self.dataset.ordered_indices() - - @property - def supports_prefetch(self): - return getattr(self.dataset, "supports_prefetch", False) - - def attr(self, attr: str, index: int): - return self.dataset.attr(attr, index) - - def prefetch(self, indices): - self.dataset.prefetch(indices) - - def get_batch_shapes(self): - return self.dataset.get_batch_shapes() - - def batch_by_size( - self, - indices, - max_tokens=None, - max_sentences=None, - required_batch_size_multiple=1, - ): - return self.dataset.batch_by_size( - indices, - max_tokens=max_tokens, - max_sentences=max_sentences, - required_batch_size_multiple=required_batch_size_multiple, - ) - - def filter_indices_by_size(self, indices, max_sizes): - return self.dataset.filter_indices_by_size(indices, max_sizes) - - @property - def can_reuse_epoch_itr_across_epochs(self): - return self.dataset.can_reuse_epoch_itr_across_epochs - - def set_epoch(self, epoch): - super().set_epoch(epoch) - if hasattr(self.dataset, "set_epoch"): - self.dataset.set_epoch(epoch) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/bucket_pad_length_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/bucket_pad_length_dataset.py deleted file mode 100644 index 0c45908017c429ac45224f447376cf27b85e042a..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/bucket_pad_length_dataset.py +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import numpy as np -import torch.nn.functional as F -from fairseq.data import BaseWrapperDataset -from fairseq.data.data_utils import get_bucketed_sizes, get_buckets - - -class BucketPadLengthDataset(BaseWrapperDataset): - """ - Bucket and pad item lengths to the nearest bucket size. This can be used to - reduce the number of unique batch shapes, which is important on TPUs since - each new batch shape requires a recompilation. - - Args: - dataset (FairseqDatset): dataset to bucket - sizes (List[int]): all item sizes - num_buckets (int): number of buckets to create - pad_idx (int): padding symbol - left_pad (bool): if True, pad on the left; otherwise right pad - """ - - def __init__( - self, - dataset, - sizes, - num_buckets, - pad_idx, - left_pad, - tensor_key=None, - ): - super().__init__(dataset) - self.pad_idx = pad_idx - self.left_pad = left_pad - - assert num_buckets > 0 - self.buckets = get_buckets(sizes, num_buckets) - self._bucketed_sizes = get_bucketed_sizes(sizes, self.buckets) - self._tensor_key = tensor_key - - def _set_tensor(self, item, val): - if self._tensor_key is None: - return val - item[self._tensor_key] = val - return item - - def _get_tensor(self, item): - if self._tensor_key is None: - return item - return item[self._tensor_key] - - def _pad(self, tensor, bucket_size, dim=-1): - num_pad = bucket_size - tensor.size(dim) - return F.pad( - tensor, - (num_pad if self.left_pad else 0, 0 if self.left_pad else num_pad), - value=self.pad_idx, - ) - - def __getitem__(self, index): - item = self.dataset[index] - bucket_size = self._bucketed_sizes[index] - tensor = self._get_tensor(item) - padded = self._pad(tensor, bucket_size) - return self._set_tensor(item, padded) - - @property - def sizes(self): - return self._bucketed_sizes - - def num_tokens(self, index): - return self._bucketed_sizes[index] - - def size(self, index): - return self._bucketed_sizes[index] diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/codedataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/codedataset.py deleted file mode 100644 index c3dc63ff6c9c1daa0c57f7f347d61933cc31c61d..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/codedataset.py +++ /dev/null @@ -1,576 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - - -import json -import logging -import os -import random -from pathlib import Path - -import numpy as np -import torch -import torch.utils.data -from fairseq.data.fairseq_dataset import FairseqDataset - -from . import data_utils - -F0_FRAME_SPACE = 0.005 # sec - - -logger = logging.getLogger(__name__) - - -class ExpressiveCodeDataConfig(object): - def __init__(self, json_path): - with open(json_path, "r") as f: - self.config = json.load(f) - self._manifests = self.config["manifests"] - - @property - def manifests(self): - return self._manifests - - @property - def n_units(self): - return self.config["n_units"] - - @property - def sampling_rate(self): - return self.config["sampling_rate"] - - @property - def code_hop_size(self): - return self.config["code_hop_size"] - - @property - def f0_stats(self): - """pre-computed f0 statistics path""" - return self.config.get("f0_stats", None) - - @property - def f0_vq_type(self): - """naive or precomp""" - return self.config["f0_vq_type"] - - @property - def f0_vq_name(self): - return self.config["f0_vq_name"] - - def get_f0_vq_naive_quantizer(self, log, norm_mean, norm_std): - key = "log" if log else "linear" - if norm_mean and norm_std: - key += "_mean_std_norm" - elif norm_mean: - key += "_mean_norm" - else: - key += "_none_norm" - return self.config["f0_vq_naive_quantizer"][key] - - @property - def f0_vq_n_units(self): - return self.config["f0_vq_n_units"] - - @property - def multispkr(self): - """how to parse speaker label from audio path""" - return self.config.get("multispkr", None) - - -def get_f0(audio, rate=16000): - try: - import amfm_decompy.basic_tools as basic - import amfm_decompy.pYAAPT as pYAAPT - from librosa.util import normalize - except ImportError: - raise "Please install amfm_decompy (`pip install AMFM-decompy`) and librosa (`pip install librosa`)." - - assert audio.ndim == 1 - frame_length = 20.0 # ms - to_pad = int(frame_length / 1000 * rate) // 2 - - audio = normalize(audio) * 0.95 - audio = np.pad(audio, (to_pad, to_pad), "constant", constant_values=0) - audio = basic.SignalObj(audio, rate) - pitch = pYAAPT.yaapt( - audio, - frame_length=frame_length, - frame_space=F0_FRAME_SPACE * 1000, - nccf_thresh1=0.25, - tda_frame_length=25.0, - ) - f0 = pitch.samp_values - return f0 - - -def interpolate_f0(f0): - try: - from scipy.interpolate import interp1d - except ImportError: - raise "Please install scipy (`pip install scipy`)" - - orig_t = np.arange(f0.shape[0]) - f0_interp = f0[:] - ii = f0_interp != 0 - if ii.sum() > 1: - f0_interp = interp1d( - orig_t[ii], f0_interp[ii], bounds_error=False, kind="linear", fill_value=0 - )(orig_t) - f0_interp = torch.Tensor(f0_interp).type_as(f0).to(f0.device) - return f0_interp - - -def naive_quantize(x, edges): - bin_idx = (x.view(-1, 1) > edges.view(1, -1)).long().sum(dim=1) - return bin_idx - - -def load_wav(full_path): - try: - import soundfile as sf - except ImportError: - raise "Please install soundfile (`pip install SoundFile`)" - data, sampling_rate = sf.read(full_path) - return data, sampling_rate - - -def parse_code(code_str, dictionary, append_eos): - code, duration = torch.unique_consecutive( - torch.ShortTensor(list(map(int, code_str.split()))), return_counts=True - ) - code = " ".join(map(str, code.tolist())) - code = dictionary.encode_line(code, append_eos).short() - - if append_eos: - duration = torch.cat((duration, duration.new_zeros((1,))), dim=0) # eos - duration = duration.short() - return code, duration - - -def parse_manifest(manifest, dictionary): - audio_files = [] - codes = [] - durations = [] - speakers = [] - - with open(manifest) as info: - for line in info.readlines(): - sample = eval(line.strip()) - if "cpc_km100" in sample: - k = "cpc_km100" - elif "hubert_km100" in sample: - k = "hubert_km100" - elif "phone" in sample: - k = "phone" - else: - assert False, "unknown format" - code = sample[k] - code, duration = parse_code(code, dictionary, append_eos=True) - - codes.append(code) - durations.append(duration) - audio_files.append(sample["audio"]) - speakers.append(sample.get("speaker", None)) - - return audio_files, codes, durations, speakers - - -def parse_speaker(path, method): - if type(path) == str: - path = Path(path) - - if method == "parent_name": - return path.parent.name - elif method == "parent_parent_name": - return path.parent.parent.name - elif method == "_": - return path.name.split("_")[0] - elif method == "single": - return "A" - elif callable(method): - return method(path) - else: - raise NotImplementedError() - - -def get_f0_by_filename(filename, tgt_sampling_rate): - audio, sampling_rate = load_wav(filename) - if sampling_rate != tgt_sampling_rate: - raise ValueError( - "{} SR doesn't match target {} SR".format(sampling_rate, tgt_sampling_rate) - ) - - # compute un-interpolated f0, and use Ann's interp in __getitem__ if set - f0 = get_f0(audio, rate=tgt_sampling_rate) - f0 = torch.from_numpy(f0.astype(np.float32)) - return f0 - - -def align_f0_to_durations(f0, durations, f0_code_ratio, tol=1): - code_len = durations.sum() - targ_len = int(f0_code_ratio * code_len) - diff = f0.size(0) - targ_len - assert abs(diff) <= tol, ( - f"Cannot subsample F0: |{f0.size(0)} - {f0_code_ratio}*{code_len}|" - f" > {tol} (dur=\n{durations})" - ) - if diff > 0: - f0 = f0[:targ_len] - elif diff < 0: - f0 = torch.cat((f0, f0.new_full((-diff,), f0[-1])), 0) - - f0_offset = 0.0 - seg_f0s = [] - for dur in durations: - f0_dur = dur.item() * f0_code_ratio - seg_f0 = f0[int(f0_offset) : int(f0_offset + f0_dur)] - seg_f0 = seg_f0[seg_f0 != 0] - if len(seg_f0) == 0: - seg_f0 = torch.tensor(0).type(seg_f0.type()) - else: - seg_f0 = seg_f0.mean() - seg_f0s.append(seg_f0) - f0_offset += f0_dur - - assert int(f0_offset) == f0.size(0), f"{f0_offset} {f0.size()} {durations.sum()}" - return torch.tensor(seg_f0s) - - -class Paddings(object): - def __init__(self, code_val, dur_val=0, f0_val=-2.0): - self.code = code_val - self.dur = dur_val - self.f0 = f0_val - - -class Shifts(object): - def __init__(self, shifts_str, pads): - self._shifts = list(map(int, shifts_str.split(","))) - assert len(self._shifts) == 2, self._shifts - assert all(s >= 0 for s in self._shifts) - self.extra_length = max(s for s in self._shifts) - self.pads = pads - - @property - def dur(self): - return self._shifts[0] - - @property - def f0(self): - return self._shifts[1] - - @staticmethod - def shift_one(seq, left_pad_num, right_pad_num, pad): - assert seq.ndim == 1 - bos = seq.new_full((left_pad_num,), pad) - eos = seq.new_full((right_pad_num,), pad) - seq = torch.cat([bos, seq, eos]) - mask = torch.ones_like(seq).bool() - mask[left_pad_num : len(seq) - right_pad_num] = 0 - return seq, mask - - def __call__(self, code, dur, f0): - if self.extra_length == 0: - code_mask = torch.zeros_like(code).bool() - dur_mask = torch.zeros_like(dur).bool() - f0_mask = torch.zeros_like(f0).bool() - return code, code_mask, dur, dur_mask, f0, f0_mask - - code, code_mask = self.shift_one(code, 0, self.extra_length, self.pads.code) - dur, dur_mask = self.shift_one( - dur, self.dur, self.extra_length - self.dur, self.pads.dur - ) - f0, f0_mask = self.shift_one( - f0, self.f0, self.extra_length - self.f0, self.pads.f0 - ) - return code, code_mask, dur, dur_mask, f0, f0_mask - - -class CodeDataset(FairseqDataset): - def __init__( - self, - manifest, - dictionary, - dur_dictionary, - f0_dictionary, - config, - discrete_dur, - discrete_f0, - log_f0, - normalize_f0_mean, - normalize_f0_std, - interpolate_f0, - return_filename=False, - strip_filename=True, - shifts="0,0", - return_continuous_f0=False, - ): - random.seed(1234) - self.dictionary = dictionary - self.dur_dictionary = dur_dictionary - self.f0_dictionary = f0_dictionary - self.config = config - - # duration config - self.discrete_dur = discrete_dur - - # pitch config - self.discrete_f0 = discrete_f0 - self.log_f0 = log_f0 - self.normalize_f0_mean = normalize_f0_mean - self.normalize_f0_std = normalize_f0_std - self.interpolate_f0 = interpolate_f0 - - self.return_filename = return_filename - self.strip_filename = strip_filename - self.f0_code_ratio = config.code_hop_size / ( - config.sampling_rate * F0_FRAME_SPACE - ) - - # use lazy loading to avoid sharing file handlers across workers - self.manifest = manifest - self._codes = None - self._durs = None - self._f0s = None - with open(f"{manifest}.leng.txt", "r") as f: - lengs = [int(line.rstrip()) for line in f] - edges = np.cumsum([0] + lengs) - self.starts, self.ends = edges[:-1], edges[1:] - with open(f"{manifest}.path.txt", "r") as f: - self.file_names = [line.rstrip() for line in f] - logger.info(f"num entries: {len(self.starts)}") - - if os.path.exists(f"{manifest}.f0_stat.pt"): - self.f0_stats = torch.load(f"{manifest}.f0_stat.pt") - elif config.f0_stats: - self.f0_stats = torch.load(config.f0_stats) - - self.multispkr = config.multispkr - if config.multispkr: - with open(f"{manifest}.speaker.txt", "r") as f: - self.spkrs = [line.rstrip() for line in f] - self.id_to_spkr = sorted(self.spkrs) - self.spkr_to_id = {k: v for v, k in enumerate(self.id_to_spkr)} - - self.pads = Paddings( - dictionary.pad(), - 0, # use 0 for duration padding - f0_dictionary.pad() if discrete_f0 else -5.0, - ) - self.shifts = Shifts(shifts, pads=self.pads) - self.return_continuous_f0 = return_continuous_f0 - - def get_data_handlers(self): - logging.info(f"loading data for {self.manifest}") - self._codes = np.load(f"{self.manifest}.code.npy", mmap_mode="r") - self._durs = np.load(f"{self.manifest}.dur.npy", mmap_mode="r") - - if self.discrete_f0: - if self.config.f0_vq_type == "precomp": - self._f0s = np.load( - f"{self.manifest}.{self.config.f0_vq_name}.npy", mmap_mode="r" - ) - elif self.config.f0_vq_type == "naive": - self._f0s = np.load(f"{self.manifest}.f0.npy", mmap_mode="r") - quantizers_path = self.config.get_f0_vq_naive_quantizer( - self.log_f0, self.normalize_f0_mean, self.normalize_f0_std - ) - quantizers = torch.load(quantizers_path) - n_units = self.config.f0_vq_n_units - self._f0_quantizer = torch.from_numpy(quantizers[n_units]) - else: - raise ValueError(f"f0_vq_type {self.config.f0_vq_type} not supported") - else: - self._f0s = np.load(f"{self.manifest}.f0.npy", mmap_mode="r") - - def preprocess_f0(self, f0, stats): - """ - 1. interpolate - 2. log transform (keep unvoiced frame 0) - """ - # TODO: change this to be dependent on config for naive quantizer - f0 = f0.clone() - if self.interpolate_f0: - f0 = interpolate_f0(f0) - - mask = f0 != 0 # only process voiced frames - if self.log_f0: - f0[mask] = f0[mask].log() - if self.normalize_f0_mean: - mean = stats["logf0_mean"] if self.log_f0 else stats["f0_mean"] - f0[mask] = f0[mask] - mean - if self.normalize_f0_std: - std = stats["logf0_std"] if self.log_f0 else stats["f0_std"] - f0[mask] = f0[mask] / std - return f0 - - def _get_raw_item(self, index): - start, end = self.starts[index], self.ends[index] - if self._codes is None: - self.get_data_handlers() - code = torch.from_numpy(np.array(self._codes[start:end])).long() - dur = torch.from_numpy(np.array(self._durs[start:end])) - f0 = torch.from_numpy(np.array(self._f0s[start:end])) - return code, dur, f0 - - def __getitem__(self, index): - code, dur, f0 = self._get_raw_item(index) - code = torch.cat([code.new([self.dictionary.bos()]), code]) - - # use 0 for eos and bos - dur = torch.cat([dur.new([0]), dur]) - if self.discrete_dur: - dur = self.dur_dictionary.encode_line( - " ".join(map(str, dur.tolist())), append_eos=False - ).long() - else: - dur = dur.float() - - # TODO: find a more elegant approach - raw_f0 = None - if self.discrete_f0: - if self.config.f0_vq_type == "precomp": - f0 = self.f0_dictionary.encode_line( - " ".join(map(str, f0.tolist())), append_eos=False - ).long() - else: - f0 = f0.float() - f0 = self.preprocess_f0(f0, self.f0_stats[self.spkrs[index]]) - if self.return_continuous_f0: - raw_f0 = f0 - raw_f0 = torch.cat([raw_f0.new([self.f0_dictionary.bos()]), raw_f0]) - f0 = naive_quantize(f0, self._f0_quantizer) - f0 = torch.cat([f0.new([self.f0_dictionary.bos()]), f0]) - else: - f0 = f0.float() - if self.multispkr: - f0 = self.preprocess_f0(f0, self.f0_stats[self.spkrs[index]]) - else: - f0 = self.preprocess_f0(f0, self.f0_stats) - f0 = torch.cat([f0.new([0]), f0]) - - if raw_f0 is not None: - *_, raw_f0, raw_f0_mask = self.shifts(code, dur, raw_f0) - else: - raw_f0_mask = None - - code, code_mask, dur, dur_mask, f0, f0_mask = self.shifts(code, dur, f0) - if raw_f0_mask is not None: - assert (raw_f0_mask == f0_mask).all() - - # is a padded frame if either input or output is padded - feats = { - "source": code[:-1], - "target": code[1:], - "mask": code_mask[1:].logical_or(code_mask[:-1]), - "dur_source": dur[:-1], - "dur_target": dur[1:], - "dur_mask": dur_mask[1:].logical_or(dur_mask[:-1]), - "f0_source": f0[:-1], - "f0_target": f0[1:], - "f0_mask": f0_mask[1:].logical_or(f0_mask[:-1]), - } - - if raw_f0 is not None: - feats["raw_f0"] = raw_f0[1:] - - if self.return_filename: - fname = self.file_names[index] - feats["filename"] = ( - fname if not self.strip_filename else Path(fname).with_suffix("").name - ) - return feats - - def __len__(self): - return len(self.starts) - - def size(self, index): - return self.ends[index] - self.starts[index] + self.shifts.extra_length - - def num_tokens(self, index): - return self.size(index) - - def collater(self, samples): - pad_idx, eos_idx = self.dictionary.pad(), self.dictionary.eos() - if len(samples) == 0: - return {} - - src_tokens = data_utils.collate_tokens( - [s["source"] for s in samples], pad_idx, eos_idx, left_pad=False - ) - - tgt_tokens = data_utils.collate_tokens( - [s["target"] for s in samples], - pad_idx=pad_idx, - eos_idx=pad_idx, # appending padding, eos is there already - left_pad=False, - ) - - src_durs, tgt_durs = [ - data_utils.collate_tokens( - [s[k] for s in samples], - pad_idx=self.pads.dur, - eos_idx=self.pads.dur, - left_pad=False, - ) - for k in ["dur_source", "dur_target"] - ] - - src_f0s, tgt_f0s = [ - data_utils.collate_tokens( - [s[k] for s in samples], - pad_idx=self.pads.f0, - eos_idx=self.pads.f0, - left_pad=False, - ) - for k in ["f0_source", "f0_target"] - ] - - mask, dur_mask, f0_mask = [ - data_utils.collate_tokens( - [s[k] for s in samples], - pad_idx=1, - eos_idx=1, - left_pad=False, - ) - for k in ["mask", "dur_mask", "f0_mask"] - ] - - src_lengths = torch.LongTensor([s["source"].numel() for s in samples]) - n_tokens = sum(len(s["source"]) for s in samples) - - result = { - "nsentences": len(samples), - "ntokens": n_tokens, - "net_input": { - "src_tokens": src_tokens, - "src_lengths": src_lengths, - "dur_src": src_durs, - "f0_src": src_f0s, - }, - "target": tgt_tokens, - "dur_target": tgt_durs, - "f0_target": tgt_f0s, - "mask": mask, - "dur_mask": dur_mask, - "f0_mask": f0_mask, - } - - if "filename" in samples[0]: - result["filename"] = [s["filename"] for s in samples] - - # TODO: remove this hack into the inference dataset - if "prefix" in samples[0]: - result["prefix"] = [s["prefix"] for s in samples] - - if "raw_f0" in samples[0]: - raw_f0s = data_utils.collate_tokens( - [s["raw_f0"] for s in samples], - pad_idx=self.pads.f0, - eos_idx=self.pads.f0, - left_pad=False, - ) - result["raw_f0"] = raw_f0s - return result diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/colorize_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/colorize_dataset.py deleted file mode 100644 index 7a6d2713791b1e80a6f5b982a4bf4ba93f6f561e..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/colorize_dataset.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import torch - -from . import BaseWrapperDataset - - -class ColorizeDataset(BaseWrapperDataset): - """Adds 'colors' property to net input that is obtained from the provided color getter for use by models""" - - def __init__(self, dataset, color_getter): - super().__init__(dataset) - self.color_getter = color_getter - - def collater(self, samples): - base_collate = super().collater(samples) - if len(base_collate) > 0: - base_collate["net_input"]["colors"] = torch.tensor( - list(self.color_getter(self.dataset, s["id"]) for s in samples), - dtype=torch.long, - ) - return base_collate diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/concat_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/concat_dataset.py deleted file mode 100644 index 01a4078bb159fa44b2d1062b9a971fe7f1abd1c2..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/concat_dataset.py +++ /dev/null @@ -1,124 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import bisect - -import numpy as np -from torch.utils.data.dataloader import default_collate - -from . import FairseqDataset - - -class ConcatDataset(FairseqDataset): - @staticmethod - def cumsum(sequence, sample_ratios): - r, s = [], 0 - for e, ratio in zip(sequence, sample_ratios): - curr_len = int(ratio * len(e)) - r.append(curr_len + s) - s += curr_len - return r - - def __init__(self, datasets, sample_ratios=1): - super(ConcatDataset, self).__init__() - assert len(datasets) > 0, "datasets should not be an empty iterable" - self.datasets = list(datasets) - if isinstance(sample_ratios, int): - sample_ratios = [sample_ratios] * len(self.datasets) - self.sample_ratios = sample_ratios - self.cumulative_sizes = self.cumsum(self.datasets, sample_ratios) - self.real_sizes = [len(d) for d in self.datasets] - - def __len__(self): - return self.cumulative_sizes[-1] - - def __getitem__(self, idx): - dataset_idx, sample_idx = self._get_dataset_and_sample_index(idx) - return self.datasets[dataset_idx][sample_idx] - - def _get_dataset_and_sample_index(self, idx: int): - dataset_idx = bisect.bisect_right(self.cumulative_sizes, idx) - if dataset_idx == 0: - sample_idx = idx - else: - sample_idx = idx - self.cumulative_sizes[dataset_idx - 1] - sample_idx = sample_idx % self.real_sizes[dataset_idx] - return dataset_idx, sample_idx - - def collater(self, samples, **extra_args): - # For now only supports datasets with same underlying collater implementations - if hasattr(self.datasets[0], "collater"): - return self.datasets[0].collater(samples, **extra_args) - else: - return default_collate(samples, **extra_args) - - def size(self, idx: int): - """ - Return an example's size as a float or tuple. - """ - dataset_idx, sample_idx = self._get_dataset_and_sample_index(idx) - return self.datasets[dataset_idx].size(sample_idx) - - def num_tokens(self, index: int): - return np.max(self.size(index)) - - def attr(self, attr: str, index: int): - dataset_idx = bisect.bisect_right(self.cumulative_sizes, index) - return getattr(self.datasets[dataset_idx], attr, None) - - @property - def sizes(self): - _dataset_sizes = [] - for ds, sr in zip(self.datasets, self.sample_ratios): - if isinstance(ds.sizes, np.ndarray): - _dataset_sizes.append(np.tile(ds.sizes, sr)) - else: - # Only support underlying dataset with single size array. - assert isinstance(ds.sizes, list) - _dataset_sizes.append(np.tile(ds.sizes[0], sr)) - return np.concatenate(_dataset_sizes) - - @property - def supports_prefetch(self): - return all(d.supports_prefetch for d in self.datasets) - - def ordered_indices(self): - """ - Returns indices sorted by length. So less padding is needed. - """ - if isinstance(self.sizes, np.ndarray) and len(self.sizes.shape) > 1: - # special handling for concatenating lang_pair_datasets - indices = np.arange(len(self)) - sizes = self.sizes - tgt_sizes = ( - sizes[:, 1] if len(sizes.shape) > 0 and sizes.shape[1] > 1 else None - ) - src_sizes = ( - sizes[:, 0] if len(sizes.shape) > 0 and sizes.shape[1] > 1 else sizes - ) - # sort by target length, then source length - if tgt_sizes is not None: - indices = indices[np.argsort(tgt_sizes[indices], kind="mergesort")] - return indices[np.argsort(src_sizes[indices], kind="mergesort")] - else: - return np.argsort(self.sizes) - - def prefetch(self, indices): - frm = 0 - for to, ds in zip(self.cumulative_sizes, self.datasets): - real_size = len(ds) - if getattr(ds, "supports_prefetch", False): - ds.prefetch([(i - frm) % real_size for i in indices if frm <= i < to]) - frm = to - - @property - def can_reuse_epoch_itr_across_epochs(self): - return all(d.can_reuse_epoch_itr_across_epochs for d in self.datasets) - - def set_epoch(self, epoch): - super().set_epoch(epoch) - for ds in self.datasets: - if hasattr(ds, "set_epoch"): - ds.set_epoch(epoch) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/concat_sentences_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/concat_sentences_dataset.py deleted file mode 100644 index 625a29370e90f9d1d7274024afb902ed83a22325..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/concat_sentences_dataset.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import torch - -from . import FairseqDataset - - -class ConcatSentencesDataset(FairseqDataset): - def __init__(self, *datasets): - super().__init__() - self.datasets = datasets - assert all( - len(ds) == len(datasets[0]) for ds in datasets - ), "datasets must have the same length" - - def __getitem__(self, index): - return torch.cat([ds[index] for ds in self.datasets]) - - def __len__(self): - return len(self.datasets[0]) - - def collater(self, samples): - return self.datasets[0].collater(samples) - - @property - def sizes(self): - return sum(ds.sizes for ds in self.datasets) - - def num_tokens(self, index): - return sum(ds.num_tokens(index) for ds in self.datasets) - - def size(self, index): - return sum(ds.size(index) for ds in self.datasets) - - def ordered_indices(self): - return self.datasets[0].ordered_indices() - - @property - def supports_prefetch(self): - return any(getattr(ds, "supports_prefetch", False) for ds in self.datasets) - - def prefetch(self, indices): - for ds in self.datasets: - if getattr(ds, "supports_prefetch", False): - ds.prefetch(indices) - - def set_epoch(self, epoch): - super().set_epoch(epoch) - for ds in self.datasets: - if hasattr(ds, "set_epoch"): - ds.set_epoch(epoch) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/data_utils.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/data_utils.py deleted file mode 100644 index 556c77c9e7a7b8733d35463361e227ab07af8be8..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/data_utils.py +++ /dev/null @@ -1,604 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -try: - from collections.abc import Iterable -except ImportError: - from collections import Iterable - -import contextlib -import itertools -import logging -import os -import re -import warnings -from typing import Optional, Tuple - -import numpy as np -import torch -from fairseq import utils -from fairseq.file_io import PathManager - -logger = logging.getLogger(__name__) - - -def infer_language_pair(path): - """Infer language pair from filename: <split>.<lang1>-<lang2>.(...).idx""" - src, dst = None, None - for filename in PathManager.ls(path): - parts = filename.split(".") - if len(parts) >= 3 and len(parts[1].split("-")) == 2: - return parts[1].split("-") - return src, dst - - -def collate_tokens( - values, - pad_idx, - eos_idx=None, - left_pad=False, - move_eos_to_beginning=False, - pad_to_length=None, - pad_to_multiple=1, - pad_to_bsz=None, -): - """Convert a list of 1d tensors into a padded 2d tensor.""" - size = max(v.size(0) for v in values) - size = size if pad_to_length is None else max(size, pad_to_length) - if pad_to_multiple != 1 and size % pad_to_multiple != 0: - size = int(((size - 0.1) // pad_to_multiple + 1) * pad_to_multiple) - - batch_size = len(values) if pad_to_bsz is None else max(len(values), pad_to_bsz) - res = values[0].new(batch_size, size).fill_(pad_idx) - - def copy_tensor(src, dst): - assert dst.numel() == src.numel() - if move_eos_to_beginning: - if eos_idx is None: - # if no eos_idx is specified, then use the last token in src - dst[0] = src[-1] - else: - dst[0] = eos_idx - dst[1:] = src[:-1] - else: - dst.copy_(src) - - for i, v in enumerate(values): - copy_tensor(v, res[i][size - len(v) :] if left_pad else res[i][: len(v)]) - return res - - -def load_indexed_dataset( - path, dictionary=None, dataset_impl=None, combine=False, default="cached" -): - """A helper function for loading indexed datasets. - - Args: - path (str): path to indexed dataset (e.g., 'data-bin/train') - dictionary (~fairseq.data.Dictionary): data dictionary - dataset_impl (str, optional): which dataset implementation to use. If - not provided, it will be inferred automatically. For legacy indexed - data we use the 'cached' implementation by default. - combine (bool, optional): automatically load and combine multiple - datasets. For example, if *path* is 'data-bin/train', then we will - combine 'data-bin/train', 'data-bin/train1', ... and return a - single ConcatDataset instance. - """ - import fairseq.data.indexed_dataset as indexed_dataset - from fairseq.data.concat_dataset import ConcatDataset - - datasets = [] - for k in itertools.count(): - path_k = path + (str(k) if k > 0 else "") - try: - path_k = indexed_dataset.get_indexed_dataset_to_local(path_k) - except Exception as e: - if "StorageException: [404] Path not found" in str(e): - logger.warning(f"path_k: {e} not found") - else: - raise e - - dataset_impl_k = dataset_impl - if dataset_impl_k is None: - dataset_impl_k = indexed_dataset.infer_dataset_impl(path_k) - dataset = indexed_dataset.make_dataset( - path_k, - impl=dataset_impl_k or default, - fix_lua_indexing=True, - dictionary=dictionary, - ) - if dataset is None: - break - logger.info("loaded {:,} examples from: {}".format(len(dataset), path_k)) - datasets.append(dataset) - if not combine: - break - if len(datasets) == 0: - return None - elif len(datasets) == 1: - return datasets[0] - else: - return ConcatDataset(datasets) - - -@contextlib.contextmanager -def numpy_seed(seed, *addl_seeds): - """Context manager which seeds the NumPy PRNG with the specified seed and - restores the state afterward""" - if seed is None: - yield - return - if len(addl_seeds) > 0: - seed = int(hash((seed, *addl_seeds)) % 1e6) - state = np.random.get_state() - np.random.seed(seed) - try: - yield - finally: - np.random.set_state(state) - - -def collect_filtered(function, iterable, filtered): - """ - Similar to :func:`filter` but collects filtered elements in ``filtered``. - - Args: - function (callable): function that returns ``False`` for elements that - should be filtered - iterable (iterable): iterable to filter - filtered (list): list to store filtered elements - """ - for el in iterable: - if function(el): - yield el - else: - filtered.append(el) - - -def _filter_by_size_dynamic(indices, size_fn, max_positions, raise_exception=False): - def compare_leq(a, b): - return a <= b if not isinstance(a, tuple) else max(a) <= b - - def check_size(idx): - if isinstance(max_positions, float) or isinstance(max_positions, int): - return size_fn(idx) <= max_positions - elif isinstance(max_positions, dict): - idx_size = size_fn(idx) - assert isinstance(idx_size, dict) - intersect_keys = set(max_positions.keys()) & set(idx_size.keys()) - return all( - all( - a is None or b is None or a <= b - for a, b in zip(idx_size[key], max_positions[key]) - ) - for key in intersect_keys - ) - else: - # For MultiCorpusSampledDataset, will generalize it later - if not isinstance(size_fn(idx), Iterable): - return all(size_fn(idx) <= b for b in max_positions) - return all( - a is None or b is None or a <= b - for a, b in zip(size_fn(idx), max_positions) - ) - - ignored = [] - itr = collect_filtered(check_size, indices, ignored) - indices = np.fromiter(itr, dtype=np.int64, count=-1) - return indices, ignored - - -def filter_by_size(indices, dataset, max_positions, raise_exception=False): - """ - [deprecated] Filter indices based on their size. - Use `FairseqDataset::filter_indices_by_size` instead. - - Args: - indices (List[int]): ordered list of dataset indices - dataset (FairseqDataset): fairseq dataset instance - max_positions (tuple): filter elements larger than this size. - Comparisons are done component-wise. - raise_exception (bool, optional): if ``True``, raise an exception if - any elements are filtered (default: False). - """ - warnings.warn( - "data_utils.filter_by_size is deprecated. " - "Use `FairseqDataset::filter_indices_by_size` instead.", - stacklevel=2, - ) - if isinstance(max_positions, float) or isinstance(max_positions, int): - if hasattr(dataset, "sizes") and isinstance(dataset.sizes, np.ndarray): - ignored = indices[dataset.sizes[indices] > max_positions].tolist() - indices = indices[dataset.sizes[indices] <= max_positions] - elif ( - hasattr(dataset, "sizes") - and isinstance(dataset.sizes, list) - and len(dataset.sizes) == 1 - ): - ignored = indices[dataset.sizes[0][indices] > max_positions].tolist() - indices = indices[dataset.sizes[0][indices] <= max_positions] - else: - indices, ignored = _filter_by_size_dynamic( - indices, dataset.size, max_positions - ) - else: - indices, ignored = _filter_by_size_dynamic(indices, dataset.size, max_positions) - - if len(ignored) > 0 and raise_exception: - raise Exception( - ( - "Size of sample #{} is invalid (={}) since max_positions={}, " - "skip this example with --skip-invalid-size-inputs-valid-test" - ).format(ignored[0], dataset.size(ignored[0]), max_positions) - ) - if len(ignored) > 0: - logger.warning( - ( - "{} samples have invalid sizes and will be skipped, " - "max_positions={}, first few sample ids={}" - ).format(len(ignored), max_positions, ignored[:10]) - ) - return indices - - -def filter_paired_dataset_indices_by_size(src_sizes, tgt_sizes, indices, max_sizes): - """Filter a list of sample indices. Remove those that are longer - than specified in max_sizes. - - Args: - indices (np.array): original array of sample indices - max_sizes (int or list[int] or tuple[int]): max sample size, - can be defined separately for src and tgt (then list or tuple) - - Returns: - np.array: filtered sample array - list: list of removed indices - """ - if max_sizes is None: - return indices, [] - if type(max_sizes) in (int, float): - max_src_size, max_tgt_size = max_sizes, max_sizes - else: - max_src_size, max_tgt_size = max_sizes - if tgt_sizes is None: - ignored = indices[src_sizes[indices] > max_src_size] - else: - ignored = indices[ - (src_sizes[indices] > max_src_size) | (tgt_sizes[indices] > max_tgt_size) - ] - if len(ignored) > 0: - if tgt_sizes is None: - indices = indices[src_sizes[indices] <= max_src_size] - else: - indices = indices[ - (src_sizes[indices] <= max_src_size) - & (tgt_sizes[indices] <= max_tgt_size) - ] - return indices, ignored.tolist() - - -def batch_by_size( - indices, - num_tokens_fn, - num_tokens_vec=None, - max_tokens=None, - max_sentences=None, - required_batch_size_multiple=1, - fixed_shapes=None, -): - """ - Yield mini-batches of indices bucketed by size. Batches may contain - sequences of different lengths. - - Args: - indices (List[int]): ordered list of dataset indices - num_tokens_fn (callable): function that returns the number of tokens at - a given index - num_tokens_vec (List[int], optional): precomputed vector of the number - of tokens for each index in indices (to enable faster batch generation) - max_tokens (int, optional): max number of tokens in each batch - (default: None). - max_sentences (int, optional): max number of sentences in each - batch (default: None). - required_batch_size_multiple (int, optional): require batch size to - be less than N or a multiple of N (default: 1). - fixed_shapes (List[Tuple[int, int]], optional): if given, batches will - only be created with the given shapes. *max_sentences* and - *required_batch_size_multiple* will be ignored (default: None). - """ - try: - from fairseq.data.data_utils_fast import ( - batch_by_size_fn, - batch_by_size_vec, - batch_fixed_shapes_fast, - ) - except ImportError: - raise ImportError( - "Please build Cython components with: " - "`python setup.py build_ext --inplace`" - ) - except ValueError: - raise ValueError( - "Please build (or rebuild) Cython components with `python setup.py build_ext --inplace`." - ) - - # added int() to avoid TypeError: an integer is required - max_tokens = int(max_tokens) if max_tokens is not None else -1 - max_sentences = max_sentences if max_sentences is not None else -1 - bsz_mult = required_batch_size_multiple - - if not isinstance(indices, np.ndarray): - indices = np.fromiter(indices, dtype=np.int64, count=-1) - - if num_tokens_vec is not None and not isinstance(num_tokens_vec, np.ndarray): - num_tokens_vec = np.fromiter(num_tokens_vec, dtype=np.int64, count=-1) - - if fixed_shapes is None: - if num_tokens_vec is None: - return batch_by_size_fn( - indices, - num_tokens_fn, - max_tokens, - max_sentences, - bsz_mult, - ) - else: - return batch_by_size_vec( - indices, - num_tokens_vec, - max_tokens, - max_sentences, - bsz_mult, - ) - - else: - fixed_shapes = np.array(fixed_shapes, dtype=np.int64) - sort_order = np.lexsort( - [ - fixed_shapes[:, 1].argsort(), # length - fixed_shapes[:, 0].argsort(), # bsz - ] - ) - fixed_shapes_sorted = fixed_shapes[sort_order] - return batch_fixed_shapes_fast(indices, num_tokens_fn, fixed_shapes_sorted) - - -def post_process(sentence: str, symbol: str): - if symbol == "sentencepiece": - sentence = sentence.replace(" ", "").replace("\u2581", " ").strip() - elif symbol == "wordpiece": - sentence = sentence.replace(" ", "").replace("_", " ").strip() - elif symbol == "letter": - sentence = sentence.replace(" ", "").replace("|", " ").strip() - elif symbol == "silence": - import re - - sentence = sentence.replace("<SIL>", "") - sentence = re.sub(" +", " ", sentence).strip() - elif symbol == "_EOW": - sentence = sentence.replace(" ", "").replace("_EOW", " ").strip() - elif symbol in {"subword_nmt", "@@ ", "@@"}: - if symbol == "subword_nmt": - symbol = "@@ " - sentence = (sentence + " ").replace(symbol, "").rstrip() - elif symbol == "none": - pass - elif symbol is not None: - raise NotImplementedError(f"Unknown post_process option: {symbol}") - return sentence - - -def compute_mask_indices( - shape: Tuple[int, int], - padding_mask: Optional[torch.Tensor], - mask_prob: float, - mask_length: int, - mask_type: str = "static", - mask_other: float = 0.0, - min_masks: int = 0, - no_overlap: bool = False, - min_space: int = 0, - require_same_masks: bool = True, - mask_dropout: float = 0.0, -) -> np.ndarray: - """ - Computes random mask spans for a given shape - - Args: - shape: the the shape for which to compute masks. - should be of size 2 where first element is batch size and 2nd is timesteps - padding_mask: optional padding mask of the same size as shape, which will prevent masking padded elements - mask_prob: probability for each token to be chosen as start of the span to be masked. this will be multiplied by - number of timesteps divided by length of mask span to mask approximately this percentage of all elements. - however due to overlaps, the actual number will be smaller (unless no_overlap is True) - mask_type: how to compute mask lengths - static = fixed size - uniform = sample from uniform distribution [mask_other, mask_length*2] - normal = sample from normal distribution with mean mask_length and stdev mask_other. mask is min 1 element - poisson = sample from possion distribution with lambda = mask length - min_masks: minimum number of masked spans - no_overlap: if false, will switch to an alternative recursive algorithm that prevents spans from overlapping - min_space: only used if no_overlap is True, this is how many elements to keep unmasked between spans - require_same_masks: if true, will randomly drop out masks until same amount of masks remains in each sample - mask_dropout: randomly dropout this percentage of masks in each example - """ - - bsz, all_sz = shape - mask = np.full((bsz, all_sz), False) - - all_num_mask = int( - # add a random number for probabilistic rounding - mask_prob * all_sz / float(mask_length) - + np.random.rand() - ) - - all_num_mask = max(min_masks, all_num_mask) - - mask_idcs = [] - for i in range(bsz): - if padding_mask is not None: - sz = all_sz - padding_mask[i].long().sum().item() - num_mask = int( - # add a random number for probabilistic rounding - mask_prob * sz / float(mask_length) - + np.random.rand() - ) - num_mask = max(min_masks, num_mask) - else: - sz = all_sz - num_mask = all_num_mask - - if mask_type == "static": - lengths = np.full(num_mask, mask_length) - elif mask_type == "uniform": - lengths = np.random.randint(mask_other, mask_length * 2 + 1, size=num_mask) - elif mask_type == "normal": - lengths = np.random.normal(mask_length, mask_other, size=num_mask) - lengths = [max(1, int(round(x))) for x in lengths] - elif mask_type == "poisson": - lengths = np.random.poisson(mask_length, size=num_mask) - lengths = [int(round(x)) for x in lengths] - else: - raise Exception("unknown mask selection " + mask_type) - - if sum(lengths) == 0: - lengths[0] = min(mask_length, sz - 1) - - if no_overlap: - mask_idc = [] - - def arrange(s, e, length, keep_length): - span_start = np.random.randint(s, e - length) - mask_idc.extend(span_start + i for i in range(length)) - - new_parts = [] - if span_start - s - min_space >= keep_length: - new_parts.append((s, span_start - min_space + 1)) - if e - span_start - length - min_space > keep_length: - new_parts.append((span_start + length + min_space, e)) - return new_parts - - parts = [(0, sz)] - min_length = min(lengths) - for length in sorted(lengths, reverse=True): - lens = np.fromiter( - (e - s if e - s >= length + min_space else 0 for s, e in parts), - np.int, - ) - l_sum = np.sum(lens) - if l_sum == 0: - break - probs = lens / np.sum(lens) - c = np.random.choice(len(parts), p=probs) - s, e = parts.pop(c) - parts.extend(arrange(s, e, length, min_length)) - mask_idc = np.asarray(mask_idc) - else: - min_len = min(lengths) - if sz - min_len <= num_mask: - min_len = sz - num_mask - 1 - - mask_idc = np.random.choice(sz - min_len, num_mask, replace=False) - - mask_idc = np.asarray( - [ - mask_idc[j] + offset - for j in range(len(mask_idc)) - for offset in range(lengths[j]) - ] - ) - - mask_idcs.append(np.unique(mask_idc[mask_idc < sz])) - - min_len = min([len(m) for m in mask_idcs]) - for i, mask_idc in enumerate(mask_idcs): - if len(mask_idc) > min_len and require_same_masks: - mask_idc = np.random.choice(mask_idc, min_len, replace=False) - if mask_dropout > 0: - num_holes = np.rint(len(mask_idc) * mask_dropout).astype(int) - mask_idc = np.random.choice( - mask_idc, len(mask_idc) - num_holes, replace=False - ) - - mask[i, mask_idc] = True - - return mask - - -def get_mem_usage(): - try: - import psutil - - mb = 1024 * 1024 - return f"used={psutil.virtual_memory().used / mb}Mb; avail={psutil.virtual_memory().available / mb}Mb" - except ImportError: - return "N/A" - - -# lens: torch.LongTensor -# returns: torch.BoolTensor -def lengths_to_padding_mask(lens): - bsz, max_lens = lens.size(0), torch.max(lens).item() - mask = torch.arange(max_lens).to(lens.device).view(1, max_lens) - mask = mask.expand(bsz, -1) >= lens.view(bsz, 1).expand(-1, max_lens) - return mask - - -# lens: torch.LongTensor -# returns: torch.BoolTensor -def lengths_to_mask(lens): - return ~lengths_to_padding_mask(lens) - - -def get_buckets(sizes, num_buckets): - buckets = np.unique( - np.percentile( - sizes, - np.linspace(0, 100, num_buckets + 1), - interpolation="lower", - )[1:] - ) - return buckets - - -def get_bucketed_sizes(orig_sizes, buckets): - sizes = np.copy(orig_sizes) - assert np.min(sizes) >= 0 - start_val = -1 - for end_val in buckets: - mask = (sizes > start_val) & (sizes <= end_val) - sizes[mask] = end_val - start_val = end_val - return sizes - - -def _find_extra_valid_paths(dataset_path: str) -> set: - paths = utils.split_paths(dataset_path) - all_valid_paths = set() - for sub_dir in paths: - contents = PathManager.ls(sub_dir) - valid_paths = [c for c in contents if re.match("valid*[0-9].*", c) is not None] - all_valid_paths |= {os.path.basename(p) for p in valid_paths} - # Remove .bin, .idx etc - roots = {os.path.splitext(p)[0] for p in all_valid_paths} - return roots - - -def raise_if_valid_subsets_unintentionally_ignored(train_cfg) -> None: - """Raises if there are paths matching 'valid*[0-9].*' which are not combined or ignored.""" - if ( - train_cfg.dataset.ignore_unused_valid_subsets - or train_cfg.dataset.combine_valid_subsets - or train_cfg.dataset.disable_validation - or not hasattr(train_cfg.task, "data") - ): - return - other_paths = _find_extra_valid_paths(train_cfg.task.data) - specified_subsets = train_cfg.dataset.valid_subset.split(",") - ignored_paths = [p for p in other_paths if p not in specified_subsets] - if ignored_paths: - advice = "Set --combine-val to combine them or --ignore-unused-valid-subsets to ignore them." - msg = f"Valid paths {ignored_paths} will be ignored. {advice}" - raise ValueError(msg) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/data_utils_fast.pyx b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/data_utils_fast.pyx deleted file mode 100644 index 1a1172ef3b2ce088baae75bbc302f5851b4fc825..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/data_utils_fast.pyx +++ /dev/null @@ -1,177 +0,0 @@ -# cython: language_level=3 -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import numpy as np - -cimport cython -cimport numpy as np -from libc.stdint cimport int32_t, int64_t -from libcpp cimport bool as bool_t - -ctypedef int64_t DTYPE_t - -@cython.cdivision(True) -@cython.boundscheck(False) -@cython.wraparound(False) -cpdef list batch_by_size_vec( - np.ndarray[int64_t, ndim=1] indices, - np.ndarray[int64_t, ndim=1] num_tokens_vec, - int64_t max_tokens, - int64_t max_sentences, - int32_t bsz_mult, -): - if indices.shape[0] == 0: - return [] - - assert max_tokens <= 0 or np.max(num_tokens_vec) <= max_tokens, ( - f"Sentences lengths should not exceed max_tokens={max_tokens}" - ) - - cdef int32_t indices_len = indices.shape[0] - cdef np.ndarray[int32_t, ndim=1] batches_ends = \ - np.zeros(indices_len, dtype=np.int32) - cdef int32_t[:] batches_ends_view = batches_ends - cdef int64_t[:] num_tokens_view = num_tokens_vec - - cdef int32_t pos = 0 - cdef int32_t new_batch_end = 0 - - cdef int64_t new_batch_max_tokens = 0 - cdef int32_t new_batch_sentences = 0 - cdef int64_t new_batch_num_tokens = 0 - - cdef bool_t overflow = False - cdef bool_t size_matches_with_bsz_mult = False - - cdef int32_t batches_count = 0 - cdef int32_t batch_start = 0 - cdef int64_t tail_max_tokens = 0 - cdef int64_t batch_max_tokens = 0 - - for pos in range(indices_len): - # At every pos we keep stats about the last complete batch [batch_start:batch_end), - # and tail [batch_end:pos]. - # 1) Every time when (batch + tail) forms a valid batch - # (according to max_tokens, max_sentences and bsz_mult) we append tail to batch. - # 2) When (batch+tail) violates max_tokens or max_sentences constraints - # we finalize running batch, and tail becomes a new batch. - # 3) There is a corner case when tail also violates constraints. - # In that situation [batch_end:pos-1] (tail without the current pos) - # gets added to the finalized batches, while [pos:pos] becomes a new tail. - # - # Important: For the sake of performance try to avoid using function calls within this loop. - - tail_max_tokens = tail_max_tokens \ - if tail_max_tokens > num_tokens_view[pos] \ - else num_tokens_view[pos] - new_batch_end = pos + 1 - new_batch_max_tokens = batch_max_tokens \ - if batch_max_tokens > tail_max_tokens \ - else tail_max_tokens - new_batch_sentences = new_batch_end - batch_start - new_batch_num_tokens = new_batch_sentences * new_batch_max_tokens - - overflow = (new_batch_sentences > max_sentences > 0 or - new_batch_num_tokens > max_tokens > 0) - size_matches_with_bsz_mult = (new_batch_sentences < bsz_mult or - new_batch_sentences % bsz_mult == 0) - - if overflow: - tail_num_tokens = tail_max_tokens * \ - (new_batch_end - batches_ends_view[batches_count]) - tail_overflow = tail_num_tokens > max_tokens > 0 - # In case of a tail overflow finalize two batches - if tail_overflow: - batches_count += 1 - batches_ends_view[batches_count] = pos - tail_max_tokens = num_tokens_view[pos] - batch_start = batches_ends_view[batches_count] - batches_count += 1 - new_batch_max_tokens = tail_max_tokens - - if overflow or size_matches_with_bsz_mult: - batches_ends_view[batches_count] = new_batch_end - batch_max_tokens = new_batch_max_tokens - tail_max_tokens = 0 - if batches_ends_view[batches_count] != indices_len: - batches_count += 1 - # Memory and time-efficient split - return np.split(indices, batches_ends[:batches_count]) - - -@cython.boundscheck(False) -@cython.wraparound(False) -cpdef list batch_by_size_fn( - np.ndarray[DTYPE_t, ndim=1] indices, - num_tokens_fn, - int64_t max_tokens, - int64_t max_sentences, - int32_t bsz_mult, -): - cdef int32_t indices_len = indices.shape[0] - cdef np.ndarray[int64_t, ndim=1] num_tokens_vec = np.zeros(indices_len, - dtype=np.int64) - cdef DTYPE_t[:] indices_view = indices - cdef DTYPE_t[:] num_tokens_vec_view = num_tokens_vec - cdef int64_t pos - for pos in range(indices_len): - num_tokens_vec[pos] = num_tokens_fn(indices_view[pos]) - return batch_by_size_vec(indices, num_tokens_vec, max_tokens, - max_sentences, bsz_mult,) - - -cdef _find_valid_shape( - DTYPE_t[:, :] shapes_view, - int64_t num_sentences, - int64_t num_tokens, -): - """Return index of first valid shape of -1 if none is found.""" - for i in range(shapes_view.shape[0]): - if num_sentences <= shapes_view[i][0] and num_tokens <= shapes_view[i][1]: - return i - return -1 - - -@cython.cdivision(True) -cpdef list batch_fixed_shapes_fast( - np.ndarray[DTYPE_t, ndim=1] indices, - num_tokens_fn, - np.ndarray[DTYPE_t, ndim=2] fixed_shapes_sorted, -): - cdef int64_t sample_len = 0 - cdef list sample_lens = [] - cdef list batch = [] - cdef list batches = [] - cdef int64_t mod_len - cdef int64_t i - cdef int64_t idx - cdef int64_t num_tokens - cdef DTYPE_t[:] indices_view = indices - cdef DTYPE_t[:, :] shapes_view = fixed_shapes_sorted - - for i in range(len(indices_view)): - idx = indices_view[i] - num_tokens = num_tokens_fn(idx) - sample_lens.append(num_tokens) - sample_len = max(sample_len, num_tokens) - - shape_idx = _find_valid_shape(shapes_view, len(batch) + 1, sample_len) - if shape_idx == -1: - batches.append(batch) - batch = [] - sample_lens = [] - sample_len = 0 - shapes_view = fixed_shapes_sorted - elif shape_idx > 0: - # small optimization for the next call to _find_valid_shape - shapes_view = shapes_view[shape_idx:] - - batch.append(idx) - - if len(batch) > 0: - batches.append(batch) - - return batches diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/denoising_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/denoising_dataset.py deleted file mode 100644 index bdb62c8d5db9c8755c72db4d0d8083c936f18dc8..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/denoising_dataset.py +++ /dev/null @@ -1,436 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import math - -import numpy as np -import torch - -from . import FairseqDataset, data_utils - - -def collate( - samples, - pad_idx, - eos_idx, - vocab, - left_pad_source=False, - left_pad_target=False, - input_feeding=True, - pad_to_length=None, -): - assert input_feeding - if len(samples) == 0: - return {} - - def merge(key, left_pad, move_eos_to_beginning=False, pad_to_length=None): - return data_utils.collate_tokens( - [s[key] for s in samples], - pad_idx, - eos_idx=None, # use eos_idx of each sample instead of vocab.eos() - left_pad=left_pad, - move_eos_to_beginning=move_eos_to_beginning, - pad_to_length=pad_to_length, - ) - - id = torch.LongTensor([s["id"] for s in samples]) - src_tokens = merge( - "source", - left_pad=left_pad_source, - pad_to_length=pad_to_length["source"] if pad_to_length is not None else None, - ) - # sort by descending source length - src_lengths = torch.LongTensor([s["source"].numel() for s in samples]) - src_lengths, sort_order = src_lengths.sort(descending=True) - id = id.index_select(0, sort_order) - src_tokens = src_tokens.index_select(0, sort_order) - - prev_output_tokens = None - target = None - if samples[0].get("target", None) is not None: - target = merge( - "target", - left_pad=left_pad_target, - pad_to_length=pad_to_length["target"] - if pad_to_length is not None - else None, - ) - target = target.index_select(0, sort_order) - ntokens = sum(len(s["target"]) for s in samples) - - if input_feeding: - # we create a shifted version of targets for feeding the - # previous output token(s) into the next decoder step - prev_output_tokens = merge( - "target", - left_pad=left_pad_target, - move_eos_to_beginning=True, - pad_to_length=pad_to_length["target"] - if pad_to_length is not None - else None, - ) - prev_output_tokens = prev_output_tokens.index_select(0, sort_order) - else: - ntokens = sum(len(s["source"]) for s in samples) - - batch = { - "id": id, - "ntokens": ntokens, - "net_input": { - "src_tokens": src_tokens, - "src_lengths": src_lengths, - }, - "target": target, - "nsentences": samples[0]["source"].size(0), - "sort_order": sort_order, - } - if prev_output_tokens is not None: - batch["net_input"]["prev_output_tokens"] = prev_output_tokens - - return batch - - -class DenoisingDataset(FairseqDataset): - """ - A wrapper around TokenBlockDataset for BART dataset. - - Args: - dataset (TokenBlockDataset): dataset to wrap - sizes (List[int]): sentence lengths - vocab (~fairseq.data.Dictionary): vocabulary - mask_idx (int): dictionary index used for masked token - mask_whole_words: only mask whole words. This should be a byte mask - over vocab indices, indicating whether it is the beginning of a - word. We will extend any mask to encompass the whole word. - shuffle (bool, optional): shuffle the elements before batching. - Default: ``True`` - seed: Seed for random number generator for reproducibility. - args: argparse arguments. - """ - - def __init__( - self, - dataset, - sizes, - vocab, - mask_idx, - mask_whole_words, - shuffle, - seed, - args, - eos=None, - item_transform_func=None, - ): - self.dataset = dataset - - self.sizes = sizes - - self.vocab = vocab - self.shuffle = shuffle - self.seed = seed - self.mask_idx = mask_idx - self.mask_whole_word = mask_whole_words - self.mask_ratio = args.mask - self.random_ratio = args.mask_random - self.insert_ratio = args.insert - self.rotate_ratio = args.rotate - self.permute_sentence_ratio = args.permute_sentences - self.eos = eos if eos is not None else vocab.eos() - self.item_transform_func = item_transform_func - - if args.bpe != "gpt2": - self.full_stop_index = self.vocab.eos() - else: - assert args.bpe == "gpt2" - self.full_stop_index = self.vocab.index("13") - - self.replace_length = args.replace_length - if self.replace_length not in [-1, 0, 1]: - raise ValueError(f"invalid arg: replace_length={self.replace_length}") - if args.mask_length not in ["subword", "word", "span-poisson"]: - raise ValueError(f"invalid arg: mask-length={args.mask_length}") - if args.mask_length == "subword" and args.replace_length not in [0, 1]: - raise ValueError(f"if using subwords, use replace-length=1 or 0") - - self.mask_span_distribution = None - if args.mask_length == "span-poisson": - _lambda = args.poisson_lambda - - lambda_to_the_k = 1 - e_to_the_minus_lambda = math.exp(-_lambda) - k_factorial = 1 - ps = [] - for k in range(0, 128): - ps.append(e_to_the_minus_lambda * lambda_to_the_k / k_factorial) - lambda_to_the_k *= _lambda - k_factorial *= k + 1 - if ps[-1] < 0.0000001: - break - ps = torch.FloatTensor(ps) - self.mask_span_distribution = torch.distributions.Categorical(ps) - - self.epoch = 0 - - @property - def can_reuse_epoch_itr_across_epochs(self): - return True # only the noise changes, not item sizes - - def set_epoch(self, epoch, **unused): - self.epoch = epoch - - def __getitem__(self, index): - with data_utils.numpy_seed(self.seed, self.epoch, index): - tokens = self.dataset[index] - assert tokens[-1] == self.eos - source, target = tokens, tokens.clone() - - if self.permute_sentence_ratio > 0.0: - source = self.permute_sentences(source, self.permute_sentence_ratio) - - if self.mask_ratio > 0: - source = self.add_whole_word_mask(source, self.mask_ratio) - - if self.insert_ratio > 0: - source = self.add_insertion_noise(source, self.insert_ratio) - - if self.rotate_ratio > 0.0 and np.random.random() < self.rotate_ratio: - source = self.add_rolling_noise(source) - # there can additional changes to make: - if self.item_transform_func is not None: - source, target = self.item_transform_func(source, target) - - assert (source >= 0).all() - assert (source[1:-1] >= 1).all() - assert (source <= len(self.vocab)).all() - assert source[0] == self.vocab.bos() - assert source[-1] == self.eos - return { - "id": index, - "source": source, - "target": target, - } - - def __len__(self): - return len(self.dataset) - - def permute_sentences(self, source, p=1.0): - full_stops = source == self.full_stop_index - # Pretend it ends with a full stop so last span is a sentence - full_stops[-2] = 1 - - # Tokens that are full stops, where the previous token is not - sentence_ends = (full_stops[1:] * ~full_stops[:-1]).nonzero(as_tuple=False) + 2 - result = source.clone() - - num_sentences = sentence_ends.size(0) - num_to_permute = math.ceil((num_sentences * 2 * p) / 2.0) - substitutions = torch.randperm(num_sentences)[:num_to_permute] - ordering = torch.arange(0, num_sentences) - ordering[substitutions] = substitutions[torch.randperm(num_to_permute)] - - # Ignore <bos> at start - index = 1 - for i in ordering: - sentence = source[(sentence_ends[i - 1] if i > 0 else 1) : sentence_ends[i]] - result[index : index + sentence.size(0)] = sentence - index += sentence.size(0) - return result - - def word_starts(self, source): - if self.mask_whole_word is not None: - is_word_start = self.mask_whole_word.gather(0, source) - else: - is_word_start = torch.ones(source.size()) - is_word_start[0] = 0 - is_word_start[-1] = 0 - return is_word_start - - def add_whole_word_mask(self, source, p): - is_word_start = self.word_starts(source) - num_to_mask = int(math.ceil(is_word_start.float().sum() * p)) - num_inserts = 0 - if num_to_mask == 0: - return source - - if self.mask_span_distribution is not None: - lengths = self.mask_span_distribution.sample(sample_shape=(num_to_mask,)) - - # Make sure we have enough to mask - cum_length = torch.cumsum(lengths, 0) - while cum_length[-1] < num_to_mask: - lengths = torch.cat( - [ - lengths, - self.mask_span_distribution.sample(sample_shape=(num_to_mask,)), - ], - dim=0, - ) - cum_length = torch.cumsum(lengths, 0) - - # Trim to masking budget - i = 0 - while cum_length[i] < num_to_mask: - i += 1 - lengths[i] = num_to_mask - (0 if i == 0 else cum_length[i - 1]) - num_to_mask = i + 1 - lengths = lengths[:num_to_mask] - - # Handle 0-length mask (inserts) separately - lengths = lengths[lengths > 0] - num_inserts = num_to_mask - lengths.size(0) - num_to_mask -= num_inserts - if num_to_mask == 0: - return self.add_insertion_noise(source, num_inserts / source.size(0)) - - assert (lengths > 0).all() - else: - lengths = torch.ones((num_to_mask,)).long() - assert is_word_start[-1] == 0 - word_starts = is_word_start.nonzero(as_tuple=False) - indices = word_starts[ - torch.randperm(word_starts.size(0))[:num_to_mask] - ].squeeze(1) - mask_random = torch.FloatTensor(num_to_mask).uniform_() < self.random_ratio - - source_length = source.size(0) - assert source_length - 1 not in indices - to_keep = torch.ones(source_length, dtype=torch.bool) - is_word_start[ - -1 - ] = 255 # acts as a long length, so spans don't go over the end of doc - if self.replace_length == 0: - to_keep[indices] = 0 - else: - # keep index, but replace it with [MASK] - source[indices] = self.mask_idx - source[indices[mask_random]] = torch.randint( - 1, len(self.vocab), size=(mask_random.sum(),) - ) - - if self.mask_span_distribution is not None: - assert len(lengths.size()) == 1 - assert lengths.size() == indices.size() - lengths -= 1 - while indices.size(0) > 0: - assert lengths.size() == indices.size() - lengths -= is_word_start[indices + 1].long() - uncompleted = lengths >= 0 - indices = indices[uncompleted] + 1 - mask_random = mask_random[uncompleted] - lengths = lengths[uncompleted] - if self.replace_length != -1: - # delete token - to_keep[indices] = 0 - else: - # keep index, but replace it with [MASK] - source[indices] = self.mask_idx - source[indices[mask_random]] = torch.randint( - 1, len(self.vocab), size=(mask_random.sum(),) - ) - else: - # A bit faster when all lengths are 1 - while indices.size(0) > 0: - uncompleted = is_word_start[indices + 1] == 0 - indices = indices[uncompleted] + 1 - mask_random = mask_random[uncompleted] - if self.replace_length != -1: - # delete token - to_keep[indices] = 0 - else: - # keep index, but replace it with [MASK] - source[indices] = self.mask_idx - source[indices[mask_random]] = torch.randint( - 1, len(self.vocab), size=(mask_random.sum(),) - ) - - assert source_length - 1 not in indices - - source = source[to_keep] - - if num_inserts > 0: - source = self.add_insertion_noise(source, num_inserts / source.size(0)) - - return source - - def add_permuted_noise(self, tokens, p): - num_words = len(tokens) - num_to_permute = math.ceil(((num_words * 2) * p) / 2.0) - substitutions = torch.randperm(num_words - 2)[:num_to_permute] + 1 - tokens[substitutions] = tokens[substitutions[torch.randperm(num_to_permute)]] - return tokens - - def add_rolling_noise(self, tokens): - offset = np.random.randint(1, max(1, tokens.size(-1) - 1) + 1) - tokens = torch.cat( - (tokens[0:1], tokens[offset:-1], tokens[1:offset], tokens[-1:]), - dim=0, - ) - return tokens - - def add_insertion_noise(self, tokens, p): - if p == 0.0: - return tokens - - num_tokens = len(tokens) - n = int(math.ceil(num_tokens * p)) - - noise_indices = torch.randperm(num_tokens + n - 2)[:n] + 1 - noise_mask = torch.zeros(size=(num_tokens + n,), dtype=torch.bool) - noise_mask[noise_indices] = 1 - result = torch.LongTensor(n + len(tokens)).fill_(-1) - - num_random = int(math.ceil(n * self.random_ratio)) - result[noise_indices[num_random:]] = self.mask_idx - result[noise_indices[:num_random]] = torch.randint( - low=1, high=len(self.vocab), size=(num_random,) - ) - - result[~noise_mask] = tokens - - assert (result >= 0).all() - return result - - def collater(self, samples, pad_to_length=None): - """Merge a list of samples to form a mini-batch. - Args: - samples (List[dict]): samples to collate - Returns: - dict: a mini-batch of data - """ - return collate( - samples, self.vocab.pad(), self.eos, self.vocab, pad_to_length=pad_to_length - ) - - def num_tokens(self, index): - """Return the number of tokens in a sample. This value is used to - enforce ``--max-tokens`` during batching.""" - return self.sizes[index] - - def size(self, index): - """Return an example's size as a float or tuple. This value is used when - filtering a dataset with ``--max-positions``.""" - return self.sizes[index] - - def ordered_indices(self): - """Return an ordered list of indices. Batches will be constructed based - on this order.""" - if self.shuffle: - indices = np.random.permutation(len(self)) - else: - indices = np.arange(len(self)) - return indices[np.argsort(self.sizes[indices], kind="mergesort")] - - def prefetch(self, indices): - self.src.prefetch(indices) - self.tgt.prefetch(indices) - - @property - def supports_prefetch(self): - return ( - hasattr(self.src, "supports_prefetch") - and self.src.supports_prefetch - and hasattr(self.tgt, "supports_prefetch") - and self.tgt.supports_prefetch - ) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/dictionary.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/dictionary.py deleted file mode 100644 index d6495389f0102156f0b2dc6f892946d572911bbe..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/dictionary.py +++ /dev/null @@ -1,401 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import os -from collections import Counter -from multiprocessing import Pool - -import torch -from fairseq import utils -from fairseq.data import data_utils -from fairseq.file_chunker_utils import Chunker, find_offsets -from fairseq.file_io import PathManager -from fairseq.tokenizer import tokenize_line - - -class Dictionary: - """A mapping from symbols to consecutive integers""" - - def __init__( - self, - *, # begin keyword-only arguments - bos="<s>", - pad="<pad>", - eos="</s>", - unk="<unk>", - extra_special_symbols=None, - ): - self.bos_word, self.unk_word, self.pad_word, self.eos_word = bos, unk, pad, eos - self.symbols = [] - self.count = [] - self.indices = {} - self.bos_index = self.add_symbol(bos) - self.pad_index = self.add_symbol(pad) - self.eos_index = self.add_symbol(eos) - self.unk_index = self.add_symbol(unk) - if extra_special_symbols: - for s in extra_special_symbols: - self.add_symbol(s) - self.nspecial = len(self.symbols) - - def __eq__(self, other): - return self.indices == other.indices - - def __getitem__(self, idx): - if idx < len(self.symbols): - return self.symbols[idx] - return self.unk_word - - def get_count(self, idx): - return self.count[idx] - - def __len__(self): - """Returns the number of symbols in the dictionary""" - return len(self.symbols) - - def __contains__(self, sym): - return sym in self.indices - - def index(self, sym): - """Returns the index of the specified symbol""" - assert isinstance(sym, str) - if sym in self.indices: - return self.indices[sym] - return self.unk_index - - def string( - self, - tensor, - bpe_symbol=None, - escape_unk=False, - extra_symbols_to_ignore=None, - unk_string=None, - include_eos=False, - separator=" ", - ): - """Helper for converting a tensor of token indices to a string. - - Can optionally remove BPE symbols or escape <unk> words. - """ - if torch.is_tensor(tensor) and tensor.dim() == 2: - return "\n".join( - self.string( - t, - bpe_symbol, - escape_unk, - extra_symbols_to_ignore, - include_eos=include_eos, - ) - for t in tensor - ) - - extra_symbols_to_ignore = set(extra_symbols_to_ignore or []) - if not include_eos: - extra_symbols_to_ignore.add(self.eos()) - - def token_string(i): - if i == self.unk(): - if unk_string is not None: - return unk_string - else: - return self.unk_string(escape_unk) - else: - return self[i] - - if hasattr(self, "bos_index"): - extra_symbols_to_ignore.add(self.bos()) - - sent = separator.join( - token_string(i) - for i in tensor - if utils.item(i) not in extra_symbols_to_ignore - ) - - return data_utils.post_process(sent, bpe_symbol) - - def unk_string(self, escape=False): - """Return unknown string, optionally escaped as: <<unk>>""" - if escape: - return "<{}>".format(self.unk_word) - else: - return self.unk_word - - def add_symbol(self, word, n=1, overwrite=False): - """Adds a word to the dictionary""" - if word in self.indices and not overwrite: - idx = self.indices[word] - self.count[idx] = self.count[idx] + n - return idx - else: - idx = len(self.symbols) - self.indices[word] = idx - self.symbols.append(word) - self.count.append(n) - return idx - - def update(self, new_dict): - """Updates counts from new dictionary.""" - for word in new_dict.symbols: - idx2 = new_dict.indices[word] - if word in self.indices: - idx = self.indices[word] - self.count[idx] = self.count[idx] + new_dict.count[idx2] - else: - idx = len(self.symbols) - self.indices[word] = idx - self.symbols.append(word) - self.count.append(new_dict.count[idx2]) - - def finalize(self, threshold=-1, nwords=-1, padding_factor=8): - """Sort symbols by frequency in descending order, ignoring special ones. - - Args: - - threshold defines the minimum word count - - nwords defines the total number of words in the final dictionary, - including special symbols - - padding_factor can be used to pad the dictionary size to be a - multiple of 8, which is important on some hardware (e.g., Nvidia - Tensor Cores). - """ - if nwords <= 0: - nwords = len(self) - - new_indices = dict(zip(self.symbols[: self.nspecial], range(self.nspecial))) - new_symbols = self.symbols[: self.nspecial] - new_count = self.count[: self.nspecial] - - c = Counter( - dict( - sorted(zip(self.symbols[self.nspecial :], self.count[self.nspecial :])) - ) - ) - for symbol, count in c.most_common(nwords - self.nspecial): - if count >= threshold: - new_indices[symbol] = len(new_symbols) - new_symbols.append(symbol) - new_count.append(count) - else: - break - - assert len(new_symbols) == len(new_indices) - - self.count = list(new_count) - self.symbols = list(new_symbols) - self.indices = new_indices - - self.pad_to_multiple_(padding_factor) - - def pad_to_multiple_(self, padding_factor): - """Pad Dictionary size to be a multiple of *padding_factor*.""" - if padding_factor > 1: - i = 0 - while len(self) % padding_factor != 0: - symbol = "madeupword{:04d}".format(i) - self.add_symbol(symbol, n=0) - i += 1 - - def bos(self): - """Helper to get index of beginning-of-sentence symbol""" - return self.bos_index - - def pad(self): - """Helper to get index of pad symbol""" - return self.pad_index - - def eos(self): - """Helper to get index of end-of-sentence symbol""" - return self.eos_index - - def unk(self): - """Helper to get index of unk symbol""" - return self.unk_index - - @classmethod - def load(cls, f): - """Loads the dictionary from a text file with the format: - - ``` - <symbol0> <count0> - <symbol1> <count1> - ... - ``` - """ - d = cls() - d.add_from_file(f) - return d - - def add_from_file(self, f): - """ - Loads a pre-existing dictionary from a text file and adds its symbols - to this instance. - """ - if isinstance(f, str): - try: - with open(PathManager.get_local_path(f), "r", encoding="utf-8") as fd: - self.add_from_file(fd) - except FileNotFoundError as fnfe: - raise fnfe - except UnicodeError: - raise Exception( - "Incorrect encoding detected in {}, please " - "rebuild the dataset".format(f) - ) - return - - lines = f.readlines() - indices_start_line = self._load_meta(lines) - - for line in lines[indices_start_line:]: - try: - line, field = line.rstrip().rsplit(" ", 1) - if field == "#fairseq:overwrite": - overwrite = True - line, field = line.rsplit(" ", 1) - else: - overwrite = False - count = int(field) - word = line - if word in self and not overwrite: - raise RuntimeError( - "Duplicate word found when loading Dictionary: '{}'. " - "Duplicate words can overwrite earlier ones by adding the " - "#fairseq:overwrite flag at the end of the corresponding row " - "in the dictionary file. If using the Camembert model, please " - "download an updated copy of the model file.".format(word) - ) - self.add_symbol(word, n=count, overwrite=overwrite) - except ValueError: - raise ValueError( - f"Incorrect dictionary format, expected '<token> <cnt> [flags]': \"{line}\"" - ) - - def _save(self, f, kv_iterator): - if isinstance(f, str): - PathManager.mkdirs(os.path.dirname(f)) - with PathManager.open(f, "w", encoding="utf-8") as fd: - return self.save(fd) - for k, v in kv_iterator: - print("{} {}".format(k, v), file=f) - - def _get_meta(self): - return [], [] - - def _load_meta(self, lines): - return 0 - - def save(self, f): - """Stores dictionary into a text file""" - ex_keys, ex_vals = self._get_meta() - self._save( - f, - zip( - ex_keys + self.symbols[self.nspecial :], - ex_vals + self.count[self.nspecial :], - ), - ) - - def dummy_sentence(self, length): - t = torch.Tensor(length).uniform_(self.nspecial + 1, len(self)).long() - t[-1] = self.eos() - return t - - def encode_line( - self, - line, - line_tokenizer=tokenize_line, - add_if_not_exist=True, - consumer=None, - append_eos=True, - reverse_order=False, - ) -> torch.IntTensor: - words = line_tokenizer(line) - if reverse_order: - words = list(reversed(words)) - nwords = len(words) - ids = torch.IntTensor(nwords + 1 if append_eos else nwords) - - for i, word in enumerate(words): - if add_if_not_exist: - idx = self.add_symbol(word) - else: - idx = self.index(word) - if consumer is not None: - consumer(word, idx) - ids[i] = idx - if append_eos: - ids[nwords] = self.eos_index - return ids - - @staticmethod - def _add_file_to_dictionary_single_worker( - filename, - tokenize, - eos_word, - start_offset, - end_offset, - ): - counter = Counter() - with Chunker(filename, start_offset, end_offset) as line_iterator: - for line in line_iterator: - for word in tokenize(line): - counter.update([word]) - counter.update([eos_word]) - return counter - - @staticmethod - def add_file_to_dictionary(filename, dict, tokenize, num_workers): - def merge_result(counter): - for w, c in sorted(counter.items()): - dict.add_symbol(w, c) - - local_file = PathManager.get_local_path(filename) - offsets = find_offsets(local_file, num_workers) - if num_workers > 1: - chunks = zip(offsets, offsets[1:]) - pool = Pool(processes=num_workers) - results = [] - for (start_offset, end_offset) in chunks: - results.append( - pool.apply_async( - Dictionary._add_file_to_dictionary_single_worker, - ( - local_file, - tokenize, - dict.eos_word, - start_offset, - end_offset, - ), - ) - ) - pool.close() - pool.join() - for r in results: - merge_result(r.get()) - else: - merge_result( - Dictionary._add_file_to_dictionary_single_worker( - local_file, tokenize, dict.eos_word, offsets[0], offsets[1] - ) - ) - - -class TruncatedDictionary(object): - def __init__(self, wrapped_dict, length): - self.__class__ = type( - wrapped_dict.__class__.__name__, - (self.__class__, wrapped_dict.__class__), - {}, - ) - self.__dict__ = wrapped_dict.__dict__ - self.wrapped_dict = wrapped_dict - self.length = min(len(self.wrapped_dict), length) - - def __len__(self): - return self.length - - def __getitem__(self, i): - if i < self.length: - return self.wrapped_dict[i] - return self.wrapped_dict.unk() diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/__init__.py deleted file mode 100644 index 074367b6a6557e4935871fa67da0f3579d52220f..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/__init__.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - - -import importlib -import os - -from fairseq import registry - -build_tokenizer, register_tokenizer, TOKENIZER_REGISTRY, _ = registry.setup_registry( - "--tokenizer", - default=None, -) - - -build_bpe, register_bpe, BPE_REGISTRY, _ = registry.setup_registry( - "--bpe", - default=None, -) - - -# automatically import any Python files in the encoders/ directory -for file in sorted(os.listdir(os.path.dirname(__file__))): - if file.endswith(".py") and not file.startswith("_"): - module = file[: file.find(".py")] - importlib.import_module("fairseq.data.encoders." + module) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/byte_bpe.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/byte_bpe.py deleted file mode 100644 index 31e3a0627827f19ca7f0b58da45e46d40a80c3bf..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/byte_bpe.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - - -from dataclasses import dataclass, field - -from fairseq import file_utils -from fairseq.data.encoders import register_bpe -from fairseq.data.encoders.byte_utils import ( - SPACE, - SPACE_ESCAPE, - byte_encode, - smart_byte_decode, -) -from fairseq.dataclass import FairseqDataclass - - -@dataclass -class ByteBpeConfig(FairseqDataclass): - sentencepiece_model_path: str = field( - default="???", metadata={"help": "path to sentencepiece model"} - ) - - -@register_bpe("byte_bpe", dataclass=ByteBpeConfig) -class ByteBPE(object): - def __init__(self, cfg): - vocab = file_utils.cached_path(cfg.sentencepiece_model_path) - try: - import sentencepiece as spm - - self.sp = spm.SentencePieceProcessor() - self.sp.Load(vocab) - except ImportError: - raise ImportError( - "Please install sentencepiece with: pip install sentencepiece" - ) - - def encode(self, x: str) -> str: - byte_encoded = byte_encode(x) - return SPACE.join(self.sp.EncodeAsPieces(byte_encoded)) - - @staticmethod - def decode(x: str) -> str: - unescaped = x.replace(SPACE, "").replace(SPACE_ESCAPE, SPACE) - return smart_byte_decode(unescaped) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/byte_utils.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/byte_utils.py deleted file mode 100644 index 94d2fe82f08e1b1d4356b13cba57a88adfcfe75d..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/byte_utils.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import re - -WHITESPACE_NORMALIZER = re.compile(r"\s+") -SPACE = chr(32) -SPACE_ESCAPE = chr(9601) -# excluding non-breaking space (160) here -PRINTABLE_LATIN = set( - list(range(32, 126 + 1)) + list(range(161, 172 + 1)) + list(range(174, 255 + 1)) -) -BYTE_TO_BCHAR = { - b: chr(b) if b in PRINTABLE_LATIN else chr(256 + b) for b in range(256) -} -BCHAR_TO_BYTE = {bc: b for b, bc in BYTE_TO_BCHAR.items()} - - -def byte_encode(x: str) -> str: - normalized = WHITESPACE_NORMALIZER.sub(SPACE, x) - return "".join([BYTE_TO_BCHAR[b] for b in normalized.encode("utf-8")]) - - -def byte_decode(x: str) -> str: - try: - return bytes([BCHAR_TO_BYTE[bc] for bc in x]).decode("utf-8") - except ValueError: - return "" - - -def smart_byte_decode(x: str) -> str: - output = byte_decode(x) - if output == "": - # DP the best recovery (max valid chars) if it's broken - n_bytes = len(x) - f = [0 for _ in range(n_bytes + 1)] - pt = [0 for _ in range(n_bytes + 1)] - for i in range(1, n_bytes + 1): - f[i], pt[i] = f[i - 1], i - 1 - for j in range(1, min(4, i) + 1): - if f[i - j] + 1 > f[i] and len(byte_decode(x[i - j : i])) > 0: - f[i], pt[i] = f[i - j] + 1, i - j - cur_pt = n_bytes - while cur_pt > 0: - if f[cur_pt] == f[pt[cur_pt]] + 1: - output = byte_decode(x[pt[cur_pt] : cur_pt]) + output - cur_pt = pt[cur_pt] - return output diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/bytes.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/bytes.py deleted file mode 100644 index f88f8f6929f5b6bdb0db470be9ebedf8fe1f752d..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/bytes.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - - -from fairseq.data.encoders import register_bpe -from fairseq.data.encoders.byte_utils import ( - SPACE, - SPACE_ESCAPE, - byte_encode, - smart_byte_decode, -) - - -@register_bpe("bytes") -class Bytes(object): - def __init__(self, *unused): - pass - - @staticmethod - def add_args(parser): - pass - - @staticmethod - def encode(x: str) -> str: - encoded = byte_encode(x) - escaped = encoded.replace(SPACE, SPACE_ESCAPE) - return SPACE.join(list(escaped)) - - @staticmethod - def decode(x: str) -> str: - unescaped = x.replace(SPACE, "").replace(SPACE_ESCAPE, SPACE) - return smart_byte_decode(unescaped) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/characters.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/characters.py deleted file mode 100644 index 24df1c78cce862244d6a037db1c5f0e91b49c04e..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/characters.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - - -from fairseq.data.encoders import register_bpe - -SPACE = chr(32) -SPACE_ESCAPE = chr(9601) - - -@register_bpe("characters") -class Characters(object): - def __init__(self, *unused): - pass - - @staticmethod - def add_args(parser): - pass - - @staticmethod - def encode(x: str) -> str: - escaped = x.replace(SPACE, SPACE_ESCAPE) - return SPACE.join(list(escaped)) - - @staticmethod - def decode(x: str) -> str: - return x.replace(SPACE, "").replace(SPACE_ESCAPE, SPACE) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/fastbpe.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/fastbpe.py deleted file mode 100644 index f7c21039549ea002e73d1ad7cde5735f215f11ee..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/fastbpe.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from dataclasses import dataclass, field - -from fairseq import file_utils -from fairseq.data.encoders import register_bpe -from fairseq.dataclass import FairseqDataclass - - -@dataclass -class fastBPEConfig(FairseqDataclass): - bpe_codes: str = field(default="???", metadata={"help": "path to fastBPE BPE"}) - - -@register_bpe("fastbpe", dataclass=fastBPEConfig) -class fastBPE(object): - def __init__(self, cfg): - if cfg.bpe_codes is None: - raise ValueError("--bpe-codes is required for --bpe=fastbpe") - codes = file_utils.cached_path(cfg.bpe_codes) - try: - import fastBPE - - self.bpe = fastBPE.fastBPE(codes) - self.bpe_symbol = "@@ " - except ImportError: - raise ImportError("Please install fastBPE with: pip install fastBPE") - - def encode(self, x: str) -> str: - return self.bpe.apply([x])[0] - - def decode(self, x: str) -> str: - return (x + " ").replace(self.bpe_symbol, "").rstrip() diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/gpt2_bpe.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/gpt2_bpe.py deleted file mode 100644 index 425ab119e1fd97157c35964410103f6f308234b8..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/gpt2_bpe.py +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from dataclasses import dataclass, field - -from fairseq import file_utils -from fairseq.data.encoders import register_bpe -from fairseq.dataclass import FairseqDataclass - -from .gpt2_bpe_utils import get_encoder - -DEFAULT_ENCODER_JSON = "https://dl.fbaipublicfiles.com/fairseq/gpt2_bpe/encoder.json" -DEFAULT_VOCAB_BPE = "https://dl.fbaipublicfiles.com/fairseq/gpt2_bpe/vocab.bpe" - - -@dataclass -class GPT2BPEConfig(FairseqDataclass): - gpt2_encoder_json: str = field( - default=DEFAULT_ENCODER_JSON, metadata={"help": "path to encoder.json"} - ) - gpt2_vocab_bpe: str = field( - default=DEFAULT_VOCAB_BPE, metadata={"help": "path to vocab.bpe"} - ) - - -@register_bpe("gpt2", dataclass=GPT2BPEConfig) -class GPT2BPE(object): - def __init__(self, cfg): - encoder_json = file_utils.cached_path(cfg.gpt2_encoder_json) - vocab_bpe = file_utils.cached_path(cfg.gpt2_vocab_bpe) - self.bpe = get_encoder(encoder_json, vocab_bpe) - - def encode(self, x: str) -> str: - return " ".join(map(str, self.bpe.encode(x))) - - def decode(self, x: str) -> str: - return self.bpe.decode( - [int(tok) if tok not in {"<unk>", "<mask>"} else tok for tok in x.split()] - ) - - def is_beginning_of_word(self, x: str) -> bool: - return self.decode(x).startswith(" ") diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/gpt2_bpe_utils.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/gpt2_bpe_utils.py deleted file mode 100644 index 996d3d4a1175fb3841242c286f08faf43fc06e3e..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/gpt2_bpe_utils.py +++ /dev/null @@ -1,140 +0,0 @@ -""" -Byte pair encoding utilities from GPT-2. - -Original source: https://github.com/openai/gpt-2/blob/master/src/encoder.py -Original license: MIT -""" - -import json -from functools import lru_cache - - -@lru_cache() -def bytes_to_unicode(): - """ - Returns list of utf-8 byte and a corresponding list of unicode strings. - The reversible bpe codes work on unicode strings. - This means you need a large # of unicode characters in your vocab if you want to avoid UNKs. - When you're at something like a 10B token dataset you end up needing around 5K for decent coverage. - This is a signficant percentage of your normal, say, 32K bpe vocab. - To avoid that, we want lookup tables between utf-8 bytes and unicode strings. - And avoids mapping to whitespace/control characters the bpe code barfs on. - """ - bs = ( - list(range(ord("!"), ord("~") + 1)) - + list(range(ord("¡"), ord("¬") + 1)) - + list(range(ord("®"), ord("ÿ") + 1)) - ) - cs = bs[:] - n = 0 - for b in range(2**8): - if b not in bs: - bs.append(b) - cs.append(2**8 + n) - n += 1 - cs = [chr(n) for n in cs] - return dict(zip(bs, cs)) - - -def get_pairs(word): - """Return set of symbol pairs in a word. - Word is represented as tuple of symbols (symbols being variable-length strings). - """ - pairs = set() - prev_char = word[0] - for char in word[1:]: - pairs.add((prev_char, char)) - prev_char = char - return pairs - - -class Encoder: - def __init__(self, encoder, bpe_merges, errors="replace"): - self.encoder = encoder - self.decoder = {v: k for k, v in self.encoder.items()} - self.errors = errors # how to handle errors in decoding - self.byte_encoder = bytes_to_unicode() - self.byte_decoder = {v: k for k, v in self.byte_encoder.items()} - self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges)))) - self.cache = {} - - try: - import regex as re - - self.re = re - except ImportError: - raise ImportError("Please install regex with: pip install regex") - - # Should haved added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions - self.pat = self.re.compile( - r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""" - ) - - def bpe(self, token): - if token in self.cache: - return self.cache[token] - word = tuple(token) - pairs = get_pairs(word) - - if not pairs: - return token - - while True: - bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf"))) - if bigram not in self.bpe_ranks: - break - first, second = bigram - new_word = [] - i = 0 - while i < len(word): - try: - j = word.index(first, i) - new_word.extend(word[i:j]) - i = j - except: - new_word.extend(word[i:]) - break - - if word[i] == first and i < len(word) - 1 and word[i + 1] == second: - new_word.append(first + second) - i += 2 - else: - new_word.append(word[i]) - i += 1 - new_word = tuple(new_word) - word = new_word - if len(word) == 1: - break - else: - pairs = get_pairs(word) - word = " ".join(word) - self.cache[token] = word - return word - - def encode(self, text): - bpe_tokens = [] - for token in self.re.findall(self.pat, text): - token = "".join(self.byte_encoder[b] for b in token.encode("utf-8")) - bpe_tokens.extend( - self.encoder[bpe_token] for bpe_token in self.bpe(token).split(" ") - ) - return bpe_tokens - - def decode(self, tokens): - text = "".join([self.decoder.get(token, token) for token in tokens]) - text = bytearray([self.byte_decoder[c] for c in text]).decode( - "utf-8", errors=self.errors - ) - return text - - -def get_encoder(encoder_json_path, vocab_bpe_path): - with open(encoder_json_path, "r") as f: - encoder = json.load(f) - with open(vocab_bpe_path, "r", encoding="utf-8") as f: - bpe_data = f.read() - bpe_merges = [tuple(merge_str.split()) for merge_str in bpe_data.split("\n")[1:-1]] - return Encoder( - encoder=encoder, - bpe_merges=bpe_merges, - ) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/hf_bert_bpe.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/hf_bert_bpe.py deleted file mode 100644 index a41c059343ec7e2914b2c9d2f53f526c33f9659d..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/hf_bert_bpe.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from dataclasses import dataclass, field -from typing import Optional - -from fairseq.data.encoders import register_bpe -from fairseq.dataclass import FairseqDataclass - - -@dataclass -class BertBPEConfig(FairseqDataclass): - bpe_cased: bool = field(default=False, metadata={"help": "set for cased BPE"}) - bpe_vocab_file: Optional[str] = field( - default=None, metadata={"help": "bpe vocab file"} - ) - - -@register_bpe("bert", dataclass=BertBPEConfig) -class BertBPE(object): - def __init__(self, cfg): - try: - from transformers import BertTokenizer - except ImportError: - raise ImportError( - "Please install transformers with: pip install transformers" - ) - - if cfg.bpe_vocab_file: - self.bert_tokenizer = BertTokenizer( - cfg.bpe_vocab_file, do_lower_case=not cfg.bpe_cased - ) - else: - vocab_file_name = ( - "bert-base-cased" if cfg.bpe_cased else "bert-base-uncased" - ) - self.bert_tokenizer = BertTokenizer.from_pretrained(vocab_file_name) - - def encode(self, x: str) -> str: - return " ".join(self.bert_tokenizer.tokenize(x)) - - def decode(self, x: str) -> str: - return self.bert_tokenizer.clean_up_tokenization( - self.bert_tokenizer.convert_tokens_to_string(x.split(" ")) - ) - - def is_beginning_of_word(self, x: str) -> bool: - return not x.startswith("##") diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/hf_byte_bpe.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/hf_byte_bpe.py deleted file mode 100644 index d92787a82628319fe02dd3b27a1d9c6b20b82718..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/hf_byte_bpe.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from dataclasses import dataclass, field - -from fairseq import file_utils -from fairseq.data.encoders import register_bpe -from fairseq.dataclass import FairseqDataclass - - -@dataclass -class HuggingFaceByteLevelBPEConfig(FairseqDataclass): - bpe_merges: str = field(default="???", metadata={"help": "path to merges.txt"}) - bpe_vocab: str = field(default="???", metadata={"help": "path to vocab.json"}) - bpe_add_prefix_space: bool = field( - default=False, metadata={"help": "add prefix space before encoding"} - ) - - -@register_bpe("hf_byte_bpe", dataclass=HuggingFaceByteLevelBPEConfig) -class HuggingFaceByteLevelBPE(object): - def __init__(self, cfg): - try: - from tokenizers import ByteLevelBPETokenizer - except ImportError: - raise ImportError( - "Please install huggingface/tokenizers with: " "pip install tokenizers" - ) - - bpe_vocab = file_utils.cached_path(cfg.bpe_vocab) - bpe_merges = file_utils.cached_path(cfg.bpe_merges) - - self.bpe = ByteLevelBPETokenizer( - bpe_vocab, - bpe_merges, - add_prefix_space=cfg.bpe_add_prefix_space, - ) - - def encode(self, x: str) -> str: - return " ".join(map(str, self.bpe.encode(x).ids)) - - def decode(self, x: str) -> str: - return self.bpe.decode( - [int(tok) if tok not in {"<unk>", "<mask>"} else tok for tok in x.split()] - ) - - def is_beginning_of_word(self, x: str) -> bool: - return self.decode(x).startswith(" ") diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/moses_tokenizer.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/moses_tokenizer.py deleted file mode 100644 index afff34eafdadaaa465bdd638ef1698785b1d184e..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/moses_tokenizer.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from dataclasses import dataclass, field - -from fairseq.data.encoders import register_tokenizer -from fairseq.dataclass import FairseqDataclass - - -@dataclass -class MosesTokenizerConfig(FairseqDataclass): - source_lang: str = field(default="en", metadata={"help": "source language"}) - target_lang: str = field(default="en", metadata={"help": "target language"}) - moses_no_dash_splits: bool = field( - default=False, metadata={"help": "don't apply dash split rules"} - ) - moses_no_escape: bool = field( - default=False, - metadata={"help": "don't perform HTML escaping on apostrophe, quotes, etc."}, - ) - - -@register_tokenizer("moses", dataclass=MosesTokenizerConfig) -class MosesTokenizer(object): - def __init__(self, cfg: MosesTokenizerConfig): - self.cfg = cfg - - try: - from sacremoses import MosesDetokenizer, MosesTokenizer - - self.tok = MosesTokenizer(cfg.source_lang) - self.detok = MosesDetokenizer(cfg.target_lang) - except ImportError: - raise ImportError( - "Please install Moses tokenizer with: pip install sacremoses" - ) - - def encode(self, x: str) -> str: - return self.tok.tokenize( - x, - aggressive_dash_splits=(not self.cfg.moses_no_dash_splits), - return_str=True, - escape=(not self.cfg.moses_no_escape), - ) - - def decode(self, x: str) -> str: - return self.detok.detokenize(x.split()) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/nltk_tokenizer.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/nltk_tokenizer.py deleted file mode 100644 index 0ab92377b3a23bb48384c3f7acf299612e8b0775..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/nltk_tokenizer.py +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from fairseq.data.encoders import register_tokenizer -from fairseq.dataclass import FairseqDataclass - - -@register_tokenizer("nltk", dataclass=FairseqDataclass) -class NLTKTokenizer(object): - def __init__(self, *unused): - try: - from nltk.tokenize import word_tokenize - - self.word_tokenize = word_tokenize - except ImportError: - raise ImportError("Please install nltk with: pip install nltk") - - def encode(self, x: str) -> str: - return " ".join(self.word_tokenize(x)) - - def decode(self, x: str) -> str: - return x diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/sentencepiece_bpe.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/sentencepiece_bpe.py deleted file mode 100644 index 0aa6cd7681d0c3a91a6917640972d008db8faef7..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/sentencepiece_bpe.py +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from dataclasses import dataclass, field -from typing import Optional - -from fairseq import file_utils -from fairseq.data.encoders import register_bpe -from fairseq.dataclass import FairseqDataclass - - -@dataclass -class SentencepieceConfig(FairseqDataclass): - sentencepiece_model: str = field( - default="???", metadata={"help": "path to sentencepiece model"} - ) - sentencepiece_enable_sampling: bool = field( - default=False, metadata={"help": "enable sampling"} - ) - sentencepiece_alpha: Optional[float] = field( - default=None, - metadata={ - "help": "soothing parameter for unigram sampling, " - "and merge probability for BPE-dropout" - }, - ) - - -@register_bpe("sentencepiece", dataclass=SentencepieceConfig) -class SentencepieceBPE(object): - def __init__(self, cfg): - self.enable_sampling = cfg.sentencepiece_enable_sampling - self.alpha = cfg.sentencepiece_alpha - sentencepiece_model = file_utils.cached_path(cfg.sentencepiece_model) - try: - import sentencepiece as spm - - self.sp = spm.SentencePieceProcessor() - self.sp.Load(sentencepiece_model) - except ImportError: - raise ImportError( - "Please install sentencepiece with: pip install sentencepiece" - ) - - def encode(self, x: str) -> str: - return " ".join( - self.sp.Encode( - x, out_type=str, enable_sampling=self.enable_sampling, alpha=self.alpha - ) - ) - - def decode(self, x: str) -> str: - return x.replace(" ", "").replace("\u2581", " ").strip() - - def is_beginning_of_word(self, x: str) -> bool: - if x in ["<unk>", "<s>", "</s>", "<pad>"]: - # special elements are always considered beginnings - # HACK: this logic is already present in fairseq/tasks/masked_lm.py - # but these special tokens are also contained in the sentencepiece - # vocabulary which causes duplicate special tokens. This hack makes - # sure that they are all taken into account. - return True - return x.startswith("\u2581") diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/space_tokenizer.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/space_tokenizer.py deleted file mode 100644 index 925ad41b7c1aee6738c63938c36bd3ee16dca812..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/space_tokenizer.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import re - -from fairseq.data.encoders import register_tokenizer -from fairseq.dataclass import FairseqDataclass - - -@register_tokenizer("space", dataclass=FairseqDataclass) -class SpaceTokenizer(object): - def __init__(self, *unused): - self.space_tok = re.compile(r"\s+") - - def encode(self, x: str) -> str: - return self.space_tok.sub(" ", x) - - def decode(self, x: str) -> str: - return x diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/subword_nmt_bpe.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/subword_nmt_bpe.py deleted file mode 100644 index 5d724d2730a5895ca55af2998c2ced471625b516..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/subword_nmt_bpe.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from dataclasses import dataclass, field - -from fairseq import file_utils -from fairseq.data.encoders import register_bpe -from fairseq.dataclass import FairseqDataclass - - -@dataclass -class SubwordNMTBPEConfig(FairseqDataclass): - bpe_codes: str = field(default="???", metadata={"help": "path to subword NMT BPE"}) - bpe_separator: str = field(default="@@", metadata={"help": "BPE separator"}) - - -@register_bpe("subword_nmt", dataclass=SubwordNMTBPEConfig) -class SubwordNMTBPE(object): - def __init__(self, cfg): - if cfg.bpe_codes is None: - raise ValueError("--bpe-codes is required for --bpe=subword_nmt") - codes = file_utils.cached_path(cfg.bpe_codes) - try: - from subword_nmt import apply_bpe - - bpe_parser = apply_bpe.create_parser() - bpe_args = bpe_parser.parse_args( - [ - "--codes", - codes, - "--separator", - cfg.bpe_separator, - ] - ) - self.bpe = apply_bpe.BPE( - bpe_args.codes, - bpe_args.merges, - bpe_args.separator, - None, - bpe_args.glossaries, - ) - self.bpe_symbol = bpe_args.separator + " " - except ImportError: - raise ImportError( - "Please install subword_nmt with: pip install subword-nmt" - ) - - def encode(self, x: str) -> str: - return self.bpe.process_line(x) - - def decode(self, x: str) -> str: - return (x + " ").replace(self.bpe_symbol, "").rstrip() diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/utils.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/utils.py deleted file mode 100644 index d93eb532ef84f0e2bc708b777229ab2cb76ca14b..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/encoders/utils.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import torch -from fairseq.data import encoders - - -def get_whole_word_mask(args, dictionary): - bpe = encoders.build_bpe(args) - if bpe is not None: - - def is_beginning_of_word(i): - if i < dictionary.nspecial: - # special elements are always considered beginnings - return True - tok = dictionary[i] - if tok.startswith("madeupword"): - return True - try: - return bpe.is_beginning_of_word(tok) - except ValueError: - return True - - mask_whole_words = torch.ByteTensor( - list(map(is_beginning_of_word, range(len(dictionary)))) - ) - return mask_whole_words - return None diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/fairseq_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/fairseq_dataset.py deleted file mode 100644 index 5c3a2349b9775ba110fb562d23947394d392f675..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/fairseq_dataset.py +++ /dev/null @@ -1,206 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import logging - -import numpy as np -import torch.utils.data -from fairseq.data import data_utils - -logger = logging.getLogger(__name__) - - -class EpochListening: - """Mixin for receiving updates whenever the epoch increments.""" - - @property - def can_reuse_epoch_itr_across_epochs(self): - """ - Whether we can reuse the :class:`fairseq.data.EpochBatchIterator` for - this dataset across epochs. - - This needs to return ``False`` if the sample sizes can change across - epochs, in which case we may need to regenerate batches at each epoch. - If your dataset relies in ``set_epoch`` then you should consider setting - this to ``False``. - """ - return True - - def set_epoch(self, epoch): - """Will receive the updated epoch number at the beginning of the epoch.""" - pass - - -class FairseqDataset(torch.utils.data.Dataset, EpochListening): - """A dataset that provides helpers for batching.""" - - def __getitem__(self, index): - raise NotImplementedError - - def __len__(self): - raise NotImplementedError - - def collater(self, samples): - """Merge a list of samples to form a mini-batch. - - Args: - samples (List[dict]): samples to collate - - Returns: - dict: a mini-batch suitable for forwarding with a Model - """ - raise NotImplementedError - - def num_tokens(self, index): - """Return the number of tokens in a sample. This value is used to - enforce ``--max-tokens`` during batching.""" - raise NotImplementedError - - def num_tokens_vec(self, indices): - """Return the number of tokens for a set of positions defined by indices. - This value is used to enforce ``--max-tokens`` during batching.""" - raise NotImplementedError - - def size(self, index): - """Return an example's size as a float or tuple. This value is used when - filtering a dataset with ``--max-positions``.""" - raise NotImplementedError - - def ordered_indices(self): - """Return an ordered list of indices. Batches will be constructed based - on this order.""" - return np.arange(len(self), dtype=np.int64) - - @property - def supports_prefetch(self): - """Whether this dataset supports prefetching.""" - return False - - def attr(self, attr: str, index: int): - return getattr(self, attr, None) - - def prefetch(self, indices): - """Prefetch the data required for this epoch.""" - raise NotImplementedError - - def get_batch_shapes(self): - """ - Return a list of valid batch shapes, for example:: - - [(8, 512), (16, 256), (32, 128)] - - The first dimension of each tuple is the batch size and can be ``None`` - to automatically infer the max batch size based on ``--max-tokens``. - The second dimension of each tuple is the max supported length as given - by :func:`fairseq.data.FairseqDataset.num_tokens`. - - This will be used by :func:`fairseq.data.FairseqDataset.batch_by_size` - to restrict batch shapes. This is useful on TPUs to avoid too many - dynamic shapes (and recompilations). - """ - return None - - def batch_by_size( - self, - indices, - max_tokens=None, - max_sentences=None, - required_batch_size_multiple=1, - ): - """ - Given an ordered set of indices, return batches according to - *max_tokens*, *max_sentences* and *required_batch_size_multiple*. - """ - from fairseq.data import data_utils - - fixed_shapes = self.get_batch_shapes() - if fixed_shapes is not None: - - def adjust_bsz(bsz, num_tokens): - if bsz is None: - assert max_tokens is not None, "Must specify --max-tokens" - bsz = max_tokens // num_tokens - if max_sentences is not None: - bsz = min(bsz, max_sentences) - elif ( - bsz >= required_batch_size_multiple - and bsz % required_batch_size_multiple != 0 - ): - bsz -= bsz % required_batch_size_multiple - return bsz - - fixed_shapes = np.array( - [ - [adjust_bsz(bsz, num_tokens), num_tokens] - for (bsz, num_tokens) in fixed_shapes - ] - ) - - try: - num_tokens_vec = self.num_tokens_vec(indices).astype("int64") - except NotImplementedError: - num_tokens_vec = None - - return data_utils.batch_by_size( - indices, - num_tokens_fn=self.num_tokens, - num_tokens_vec=num_tokens_vec, - max_tokens=max_tokens, - max_sentences=max_sentences, - required_batch_size_multiple=required_batch_size_multiple, - fixed_shapes=fixed_shapes, - ) - - def filter_indices_by_size(self, indices, max_sizes): - """ - Filter a list of sample indices. Remove those that are longer than - specified in *max_sizes*. - - WARNING: don't update, override method in child classes - - Args: - indices (np.array): original array of sample indices - max_sizes (int or list[int] or tuple[int]): max sample size, - can be defined separately for src and tgt (then list or tuple) - - Returns: - np.array: filtered sample array - list: list of removed indices - """ - if isinstance(max_sizes, float) or isinstance(max_sizes, int): - if hasattr(self, "sizes") and isinstance(self.sizes, np.ndarray): - ignored = indices[self.sizes[indices] > max_sizes].tolist() - indices = indices[self.sizes[indices] <= max_sizes] - elif ( - hasattr(self, "sizes") - and isinstance(self.sizes, list) - and len(self.sizes) == 1 - ): - ignored = indices[self.sizes[0][indices] > max_sizes].tolist() - indices = indices[self.sizes[0][indices] <= max_sizes] - else: - indices, ignored = data_utils._filter_by_size_dynamic( - indices, self.size, max_sizes - ) - else: - indices, ignored = data_utils._filter_by_size_dynamic( - indices, self.size, max_sizes - ) - return indices, ignored - - @property - def supports_fetch_outside_dataloader(self): - """Whether this dataset supports fetching outside the workers of the dataloader.""" - return True - - -class FairseqIterableDataset(torch.utils.data.IterableDataset, EpochListening): - """ - For datasets that need to be read sequentially, usually because the data is - being streamed or otherwise can't be manipulated on a single machine. - """ - - def __iter__(self): - raise NotImplementedError diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/fasta_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/fasta_dataset.py deleted file mode 100644 index 007011974a997fd7446dd29d7eba097d7513bab0..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/fasta_dataset.py +++ /dev/null @@ -1,107 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import os -import subprocess -import threading -from pathlib import Path - -import numpy as np -import torch - - -def fasta_file_path(prefix_path): - return prefix_path + ".fasta" - - -class FastaDataset(torch.utils.data.Dataset): - """ - For loading protein sequence datasets in the common FASTA data format - """ - - def __init__(self, path: str, cache_indices=False): - self.fn = fasta_file_path(path) - self.threadlocal = threading.local() - self.cache = Path(f"{path}.fasta.idx.npy") - if cache_indices: - if self.cache.exists(): - self.offsets, self.sizes = np.load(self.cache) - else: - self.offsets, self.sizes = self._build_index(path) - np.save(self.cache, np.stack([self.offsets, self.sizes])) - else: - self.offsets, self.sizes = self._build_index(path) - - def _get_file(self): - if not hasattr(self.threadlocal, "f"): - self.threadlocal.f = open(self.fn, "r") - return self.threadlocal.f - - def __getitem__(self, idx): - f = self._get_file() - f.seek(self.offsets[idx]) - desc = f.readline().strip() - line = f.readline() - seq = "" - while line != "" and line[0] != ">": - seq += line.strip() - line = f.readline() - return desc, seq - - def __len__(self): - return self.offsets.size - - def _build_index(self, path: str): - # Use grep and awk to get 100M/s on local SSD. - # Should process your enormous 100G fasta in ~10 min single core... - path = fasta_file_path(path) - bytes_offsets = subprocess.check_output( - f"cat {path} | tqdm --bytes --total $(wc -c < {path})" - "| grep --byte-offset '^>' -o | cut -d: -f1", - shell=True, - ) - fasta_lengths = subprocess.check_output( - f"cat {path} | tqdm --bytes --total $(wc -c < {path})" - "| awk '/^>/ {print \"\";next;} { printf(\"%s\",$0);}' | tail -n+2 | awk '{print length($1)}'", - shell=True, - ) - bytes_np = np.fromstring(bytes_offsets, dtype=np.int64, sep=" ") - sizes_np = np.fromstring(fasta_lengths, dtype=np.int64, sep=" ") - return bytes_np, sizes_np - - def __setstate__(self, state): - self.__dict__ = state - self.threadlocal = threading.local() - - def __getstate__(self): - d = {} - for i, v in self.__dict__.items(): - if i != "threadlocal": - d[i] = v - return d - - def __del__(self): - if hasattr(self.threadlocal, "f"): - self.threadlocal.f.close() - del self.threadlocal.f - - @staticmethod - def exists(path): - return os.path.exists(fasta_file_path(path)) - - -class EncodedFastaDataset(FastaDataset): - """ - The FastaDataset returns raw sequences - this allows us to return - indices with a dictionary instead. - """ - - def __init__(self, path, dictionary): - super().__init__(path, cache_indices=True) - self.dictionary = dictionary - - def __getitem__(self, idx): - desc, seq = super().__getitem__(idx) - return self.dictionary.encode_line(seq, line_tokenizer=list).long() diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/huffman/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/huffman/__init__.py deleted file mode 100644 index 9b61fafadba28f65fe78a28b2099368b83cfcf41..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/huffman/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from .huffman_coder import HuffmanCodeBuilder, HuffmanCoder -from .huffman_mmap_indexed_dataset import ( - HuffmanMMapIndex, - HuffmanMMapIndexedDataset, - HuffmanMMapIndexedDatasetBuilder, - vocab_file_path, -) - -__all__ = [ - "HuffmanCoder", - "HuffmanCodeBuilder", - "HuffmanMMapIndexedDatasetBuilder", - "HuffmanMMapIndexedDataset", - "HuffmanMMapIndex", - "vocab_file_path", -] diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/huffman/huffman_coder.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/huffman/huffman_coder.py deleted file mode 100644 index c04f84564e6a22209439c67fed3cac31f010c6e9..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/huffman/huffman_coder.py +++ /dev/null @@ -1,267 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import re -import typing as tp -from collections import Counter, deque -from dataclasses import dataclass - -from bitarray import bitarray, util -from fairseq.data import Dictionary - -# basically we have to write to addressable bytes for the memory mapped -# dataset loader. Sentences that get encoded to a length that is not a -# multiple of BLOCKSIZE (a byte) will be padded to fit. (see _pad in the coder) -BLOCKSIZE = 8 - - -class HuffmanCoder: - def __init__( - self, root: "HuffmanNode", bos="<s>", pad="<pad>", eos="</s>", unk="<unk>" - ): - self.root = root - self.table = root.code_table() - self.bos_word, self.unk_word, self.pad_word, self.eos_word = bos, unk, pad, eos - - def _pad(self, a: bitarray) -> bitarray: - """ - bitpadding, 1 then 0. - - If the array is already a multiple of blocksize, we add a full block. - """ - pad_len = BLOCKSIZE - (len(a) % BLOCKSIZE) - 1 - padding = bitarray("1" + "0" * pad_len) - return a + padding - - def _unpad(self, a: bitarray) -> bitarray: - """ - remove the bitpadding. - - There will be a set of 0s preceded by a 1 at the end of the bitarray, we remove that - """ - # count the 0 padding at the end until we find the first 1 - # we want to remove the one too - remove_cnt = util.rindex(a, 1) - return a[:remove_cnt] - - def encode(self, iter: tp.List[str]) -> bytes: - """ - encode a list of tokens a return bytes. We use bitpadding to make sure the encoded bits fit in bytes. - """ - a = bitarray() - for token in iter: - code = self.get_code(token) - if code is None: - if self.unk_word is None: - raise Exception(f"unknown token {token} cannot be encoded.") - else: - token = self.unk_word - a = a + self.get_code(token) - return self._pad(a).tobytes() - - def decode(self, bits: bytes) -> tp.Iterator["HuffmanNode"]: - """ - take bitpadded bytes and decode it to a set of leaves. You can then use each node to find the symbol/id - """ - a = bitarray() - a.frombytes(bits) - return self.root.decode(self._unpad(a)) - - def get_code(self, symbol: str) -> tp.Optional[bitarray]: - node = self.get_node(symbol) - return None if node is None else node.code - - def get_node(self, symbol: str) -> "HuffmanNode": - return self.table.get(symbol) - - @classmethod - def from_file( - cls, - filename: str, - bos="<s>", - pad="<pad>", - eos="</s>", - unk="<unk>", - ) -> "HuffmanCoder": - builder = HuffmanCodeBuilder.from_file(filename) - return builder.build_code(bos=bos, pad=pad, eos=eos, unk=unk) - - def to_file(self, filename, sep="\t"): - nodes = list(self.table.values()) - nodes.sort(key=lambda n: n.id) - with open(filename, "w", encoding="utf-8") as output: - for n in nodes: - output.write(f"{n.symbol}{sep}{n.count}\n") - - def __iter__(self): - for n in self.table.values(): - yield n - - def merge(self, other_coder: "HuffmanCoder") -> "HuffmanCoder": - builder = HuffmanCodeBuilder() - for n in self: - builder.increment(n.symbol, n.count) - for n in other_coder: - builder.increment(n.symbol, n.count) - return builder.build_code() - - def __eq__(self, other: "HuffmanCoder") -> bool: - return self.table == other.table - - def __len__(self) -> int: - return len(self.table) - - def __contains__(self, sym: str) -> bool: - return sym in self.table - - def to_dictionary(self) -> Dictionary: - dictionary = Dictionary(bos=self.bos, unk=self.unk, pad=self.pad, eos=self.eos) - for n in self: - dictionary.add_symbol(n.symbol, n=n.count) - dictionary.finalize() - return dictionary - - -@dataclass -class HuffmanNode: - """ - a node in a Huffman tree - """ - - id: int - count: int - symbol: tp.Optional[str] = None - left: tp.Optional["HuffmanNode"] = None - right: tp.Optional["HuffmanNode"] = None - code: tp.Optional[bitarray] = None - - def is_leaf(self) -> bool: - return self.left is None and self.right is None - - def code_table( - self, prefix: tp.Optional[bitarray] = None - ) -> tp.Dict[str, "HuffmanNode"]: - defaulted_prefix = prefix if prefix is not None else bitarray() - if self.is_leaf(): - self.code = ( - defaulted_prefix if len(defaulted_prefix) > 0 else bitarray("0") - ) # leaf could be the root if there is only one symbol - return {self.symbol: self} - - codes_right = self.right.code_table(defaulted_prefix + bitarray([0])) - codes_left = self.left.code_table(defaulted_prefix + bitarray([1])) - return {**codes_left, **codes_right} - - def decode(self, bits: bitarray) -> tp.Iterator["HuffmanNode"]: - current_node = self - for bit in bits: - if bit == 0: # go right - current_node = current_node.right - else: # go left - current_node = current_node.left - if current_node is None: - # we shouldn't be on a leaf here - raise Exception("fell off a leaf") - if current_node.is_leaf(): - yield current_node - current_node = self - if current_node != self: - raise Exception("couldn't decode all the bits") - - -class HuffmanCodeBuilder: - """ - build a dictionary with occurence count and then build the Huffman code for it. - """ - - def __init__(self): - self.symbols = Counter() - - def add_symbols(self, *syms) -> None: - self.symbols.update(syms) - - def increment(self, symbol: str, cnt: int) -> None: - self.symbols[symbol] += cnt - - @classmethod - def from_file(cls, filename): - c = cls() - with open(filename, "r", encoding="utf-8") as input: - for line in input: - split = re.split(r"[\s]+", line) - c.increment(split[0], int(split[1])) - return c - - def to_file(self, filename, sep="\t"): - with open(filename, "w", encoding="utf-8") as output: - for (tok, cnt) in self.symbols.most_common(): - output.write(f"{tok}{sep}{cnt}\n") - - def _smallest(self, q1: deque, q2: deque) -> HuffmanNode: - if len(q1) == 0: - return q2.pop() - - if len(q2) == 0: - return q1.pop() - - if q1[-1].count < q2[-1].count: - return q1.pop() - - return q2.pop() - - def __add__(self, c: "HuffmanCodeBuilder") -> "HuffmanCodeBuilder": - new_c = self.symbols + c.symbols - new_b = HuffmanCodeBuilder() - new_b.symbols = new_c - return new_b - - def build_code( - self, - bos="<s>", - pad="<pad>", - eos="</s>", - unk="<unk>", - ) -> HuffmanCoder: - assert len(self.symbols) > 0, "cannot build code from empty list of symbols" - - if self.symbols[bos] == 0: - self.add_symbols(bos) - if self.symbols[pad] == 0: - self.add_symbols(pad) - if self.symbols[eos] == 0: - self.add_symbols(eos) - if self.symbols[unk] == 0: - self.add_symbols(unk) - - node_id = 0 - leaves_queue = deque( - [ - HuffmanNode(symbol=symbol, count=count, id=idx) - for idx, (symbol, count) in enumerate(self.symbols.most_common()) - ] - ) # left are the most common, right are the least common - - if len(leaves_queue) == 1: - root = leaves_queue.pop() - root.id = 0 - return HuffmanCoder(root) - - nodes_queue = deque() - - while len(leaves_queue) > 0 or len(nodes_queue) != 1: - # get the lowest two nodes at the head of each queue - node1 = self._smallest(leaves_queue, nodes_queue) - node2 = self._smallest(leaves_queue, nodes_queue) - - # add new node - nodes_queue.appendleft( - HuffmanNode( - count=node1.count + node2.count, left=node1, right=node2, id=node_id - ) - ) - node_id += 1 - - # we are left with the root - return HuffmanCoder(nodes_queue.pop(), bos=bos, pad=pad, eos=eos, unk=unk) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/huffman/huffman_mmap_indexed_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/huffman/huffman_mmap_indexed_dataset.py deleted file mode 100644 index 9b098f2c2be32ef65525dd773a6664d7823ada38..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/huffman/huffman_mmap_indexed_dataset.py +++ /dev/null @@ -1,287 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import mmap -import os -import shutil -import struct -import typing as tp -from functools import lru_cache - -import numpy as np -import torch -from fairseq.data import indexed_dataset -from fairseq.data.huffman import HuffmanCoder -from fairseq.file_io import PathManager - - -class HuffmanMMapIndex: - """ - keep an index of the offsets in the huffman binary file. - First a header, then the list of sizes (num tokens) for each instance and finally - the addresses of each instance. - """ - - _HDR_MAGIC = b"HUFFIDX\x00\x00" - _VERSION = 1 - - @classmethod - def writer(cls, path: str, data_len: int): - class _Writer: - def __enter__(self): - self._file = open(path, "wb") - - # write header (magic + version) - self._file.write(cls._HDR_MAGIC) - self._file.write(struct.pack("<Q", cls._VERSION)) - self._file.write(struct.pack("<Q", data_len)) - - return self - - def write(self, sizes, pointers): - # add number of items in the index to the header - self._file.write(struct.pack("<Q", len(sizes))) - - # write sizes - sizes = np.array(sizes, dtype=np.int32) - self._file.write(sizes.tobytes(order="C")) - del sizes - - # write address pointers - pointers = np.array(pointers, dtype=np.int64) - self._file.write(pointers.tobytes(order="C")) - del pointers - - def __exit__(self, exc_type, exc_val, exc_tb): - self._file.close() - - return _Writer() - - def __init__(self, path): - with open(path, "rb") as stream: - # read headers - magic_test = stream.read(9) - assert self._HDR_MAGIC == magic_test, ( - "Index file doesn't match expected format. " - "Make sure that --dataset-impl is configured properly." - ) - (version,) = struct.unpack("<Q", stream.read(8)) - assert ( - self._VERSION == version - ), f"Unexpected file version{version} != code version {self._VERSION}" - - # read length of data file - (self._data_len,) = struct.unpack("<Q", stream.read(8)) - # read number of items in data file/index - (self._len,) = struct.unpack("<Q", stream.read(8)) - offset = stream.tell() - - indexed_dataset._warmup_mmap_file(path) - - self._bin_buffer_mmap = np.memmap(path, mode="r", order="C") - self._bin_buffer = memoryview(self._bin_buffer_mmap) - self._sizes = np.frombuffer( - self._bin_buffer, dtype=np.int32, count=self._len, offset=offset - ) - self._pointers = np.frombuffer( - self._bin_buffer, - dtype=np.int64, - count=self._len, - offset=offset + self._sizes.nbytes, - ) - - def __del__(self): - self._bin_buffer_mmap._mmap.close() - del self._bin_buffer_mmap - - def __iter__(self): - for i in range(self._len): - yield self[i] - - @property - def data_len(self): - return self._data_len - - @property - def sizes(self): - return self._sizes - - @lru_cache(maxsize=8) - def __getitem__(self, i): - return self._pointers[i], self._sizes[i] - - def __len__(self): - return self._len - - -def vocab_file_path(prefix_path): - return prefix_path + ".vocab" - - -class HuffmanMMapIndexedDataset(torch.utils.data.Dataset): - """ - an indexed dataset that use mmap and memoryview to access data from disk - that was compressed with a HuffmanCoder. - """ - - def __init__(self, prefix_path): - super().__init__() - - self._prefix_path = None - self._index = None - self._bin_buffer = None - self._coder = None - self._file = None - - self._bin_buffer_mmap = None - - self._do_init(prefix_path) - - def __getstate__(self): - return self._prefix_path - - def __setstate__(self, state): - self._do_init(state) - - def _do_init(self, prefix_path): - self._prefix_path = prefix_path - self._index = HuffmanMMapIndex( - indexed_dataset.index_file_path(self._prefix_path) - ) - self._coder = HuffmanCoder.from_file(vocab_file_path(self._prefix_path)) - - indexed_dataset._warmup_mmap_file( - indexed_dataset.data_file_path(self._prefix_path) - ) - self._file = os.open( - indexed_dataset.data_file_path(self._prefix_path), os.O_RDONLY - ) - self._bin_buffer_mmap = mmap.mmap( - self._file, - self._index.data_len, - access=mmap.ACCESS_READ, - ) - self._bin_buffer = memoryview(self._bin_buffer_mmap) - - def __del__(self): - del self._bin_buffer - if self._file: - os.close(self._file) - del self._index - - def __len__(self): - return len(self._index) - - def _decode(self, i): - ptr, _ = self._index[i] - if i == 0: - raw_bytes = self._bin_buffer[:ptr] - else: - (prev_ptr, _) = self._index[i - 1] - raw_bytes = self._bin_buffer[prev_ptr:ptr] - - return self._coder.decode(raw_bytes.tobytes()) - - @lru_cache(maxsize=8) - def __getitem__(self, i): - nodes = self._decode(i) - return torch.tensor([n.id for n in nodes], dtype=torch.int64) - - def __iter__(self): - for idx in range(len(self)): - yield self[idx] - - def get_symbols(self, i): - nodes = self._decode(i) - for n in nodes: - yield n.symbol - - @property - def sizes(self): - return self._index.sizes - - @property - def supports_prefetch(self): - return False - - @property - def coder(self): - return self._coder - - @staticmethod - def exists(prefix_path): - return ( - PathManager.exists(indexed_dataset.index_file_path(prefix_path)) - and PathManager.exists(indexed_dataset.data_file_path(prefix_path)) - and PathManager.exists(vocab_file_path(prefix_path)) - ) - - -class HuffmanMMapIndexedDatasetBuilder: - """ - Helper to build a memory mapped datasets with a huffman encoder. - You can either open/close this manually or use it as a ContextManager. - Provide your own coder, it will then be stored alongside the dataset. - The builder will first write the vocab file, then open the binary file so you can stream - into it, finally the index will be written when the builder is closed (your index should fit in memory). - """ - - def __init__(self, path_prefix: str, coder: HuffmanCoder) -> None: - self._path_prefix = path_prefix - self._coder = coder - self._sizes = [] - self._ptrs = [] - self._data_len = 0 - - def open(self): - self._coder.to_file(vocab_file_path(self._path_prefix)) - self._data_file = open(indexed_dataset.data_file_path(self._path_prefix), "wb") - - def __enter__(self) -> "HuffmanMMapIndexedDatasetBuilder": - self.open() - return self - - def add_item(self, tokens: tp.List[str]) -> None: - """ - add a list of tokens to the dataset, they will compressed with the - provided coder before being written to file. - """ - encoded = self._coder.encode(tokens) - code_len = len(encoded) - last_ptr = 0 - if len(self._ptrs) > 0: - last_ptr = self._ptrs[-1] - self._sizes.append(len(tokens)) - self._ptrs.append(last_ptr + code_len) - self._data_len += code_len - self._data_file.write(encoded) - - def append(self, other_dataset_path_prefix: str) -> None: - """ - append an existing dataset. - Beware, if it wasn't built with the same coder, you are in trouble. - """ - other_index = HuffmanMMapIndex( - indexed_dataset.index_file_path(other_dataset_path_prefix) - ) - for (ptr, size) in other_index: - self._ptrs.append(ptr + self._data_len) - self._sizes.append(size) - - # Concatenate data - with open(indexed_dataset.data_file_path(other_dataset_path_prefix), "rb") as f: - shutil.copyfileobj(f, self._data_file) - - self._data_len += other_index.data_len - - def close(self): - self._data_file.close() - with HuffmanMMapIndex.writer( - indexed_dataset.index_file_path(self._path_prefix), self._data_len - ) as index: - index.write(self._sizes, self._ptrs) - - def __exit__(self, exc_type, exc_val, exc_tb) -> None: - self.close() diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/id_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/id_dataset.py deleted file mode 100644 index 3e4d7969cf2a26e852b466f165a6fadabae3b35f..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/id_dataset.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import torch - -from . import FairseqDataset - - -class IdDataset(FairseqDataset): - def __getitem__(self, index): - return index - - def __len__(self): - return 0 - - def collater(self, samples): - return torch.tensor(samples) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/indexed_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/indexed_dataset.py deleted file mode 100644 index dc7abb149deef56e2ff2b80c13b3dfce3f3615c7..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/indexed_dataset.py +++ /dev/null @@ -1,586 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import shutil -import struct -from functools import lru_cache -from typing import Union - -import numpy as np -import torch -from fairseq.data.fasta_dataset import FastaDataset -from fairseq.data.huffman import HuffmanMMapIndex, HuffmanMMapIndexedDataset -from fairseq.dataclass.constants import DATASET_IMPL_CHOICES -from fairseq.file_io import PathManager - -from . import FairseqDataset - - -def best_fitting_int_dtype( - max_int_to_represent, -) -> Union[np.uint16, np.uint32, np.int64]: - - if max_int_to_represent is None: - return np.uint32 # Safe guess - elif max_int_to_represent < 65500: - return np.uint16 - elif max_int_to_represent < 4294967295: - return np.uint32 - else: - return np.int64 - # we avoid np.uint64 because it doesn't save space and its type promotion behaves unexpectedly - # https://github.com/numpy/numpy/issues/5745 - - -def get_available_dataset_impl(): - return list(map(str, DATASET_IMPL_CHOICES)) - - -def infer_dataset_impl(path): - if IndexedRawTextDataset.exists(path): - return "raw" - elif IndexedDataset.exists(path): - with open(index_file_path(path), "rb") as f: - magic = f.read(8) - if magic == IndexedDataset._HDR_MAGIC: - return "cached" - elif magic == MMapIndexedDataset.Index._HDR_MAGIC[:8]: - return "mmap" - elif magic == HuffmanMMapIndex._HDR_MAGIC[:8]: - return "huffman" - else: - return None - elif FastaDataset.exists(path): - return "fasta" - else: - return None - - -def make_builder(out_file, impl, vocab_size=None): - if impl == "mmap": - return MMapIndexedDatasetBuilder( - out_file, dtype=best_fitting_int_dtype(vocab_size) - ) - elif impl == "fasta": - raise NotImplementedError - elif impl == "huffman": - raise ValueError( - "Use HuffmanCodeBuilder directly as it has a different interface." - ) - else: - return IndexedDatasetBuilder(out_file) - - -def make_dataset(path, impl, fix_lua_indexing=False, dictionary=None): - if impl == "raw" and IndexedRawTextDataset.exists(path): - assert dictionary is not None - return IndexedRawTextDataset(path, dictionary) - elif impl == "lazy" and IndexedDataset.exists(path): - return IndexedDataset(path, fix_lua_indexing=fix_lua_indexing) - elif impl == "cached" and IndexedDataset.exists(path): - return IndexedCachedDataset(path, fix_lua_indexing=fix_lua_indexing) - elif impl == "mmap" and MMapIndexedDataset.exists(path): - return MMapIndexedDataset(path) - elif impl == "fasta" and FastaDataset.exists(path): - from fairseq.data.fasta_dataset import EncodedFastaDataset - - return EncodedFastaDataset(path, dictionary) - elif impl == "huffman" and HuffmanMMapIndexedDataset.exists(path): - return HuffmanMMapIndexedDataset(path) - return None - - -def dataset_exists(path, impl): - if impl == "raw": - return IndexedRawTextDataset.exists(path) - elif impl == "mmap": - return MMapIndexedDataset.exists(path) - elif impl == "huffman": - return HuffmanMMapIndexedDataset.exists(path) - else: - return IndexedDataset.exists(path) - - -def read_longs(f, n): - a = np.empty(n, dtype=np.int64) - f.readinto(a) - return a - - -def write_longs(f, a): - f.write(np.array(a, dtype=np.int64)) - - -_code_to_dtype = { - 1: np.uint8, - 2: np.int8, - 3: np.int16, - 4: np.int32, - 5: np.int64, - 6: np.float64, - 7: np.double, - 8: np.uint16, - 9: np.uint32, - 10: np.uint64, -} - - -def _dtype_header_code(dtype) -> int: - for k in _code_to_dtype.keys(): - if _code_to_dtype[k] == dtype: - return k - raise ValueError(dtype) - - -def index_file_path(prefix_path): - return prefix_path + ".idx" - - -def data_file_path(prefix_path): - return prefix_path + ".bin" - - -class IndexedDataset(FairseqDataset): - """Loader for TorchNet IndexedDataset""" - - _HDR_MAGIC = b"TNTIDX\x00\x00" - - def __init__(self, path, fix_lua_indexing=False): - super().__init__() - self.path = path - self.fix_lua_indexing = fix_lua_indexing - self.data_file = None - self.read_index(path) - - def read_index(self, path): - with open(index_file_path(path), "rb") as f: - magic = f.read(8) - assert magic == self._HDR_MAGIC, ( - "Index file doesn't match expected format. " - "Make sure that --dataset-impl is configured properly." - ) - version = f.read(8) - assert struct.unpack("<Q", version) == (1,) - code, self.element_size = struct.unpack("<QQ", f.read(16)) - self.dtype = _code_to_dtype[code] - self._len, self.s = struct.unpack("<QQ", f.read(16)) - self.dim_offsets = read_longs(f, self._len + 1) - self.data_offsets = read_longs(f, self._len + 1) - self.sizes = read_longs(f, self.s) - - def read_data(self, path): - self.data_file = open(data_file_path(path), "rb", buffering=0) - - def check_index(self, i): - if i < 0 or i >= self._len: - raise IndexError("index out of range") - - def __del__(self): - if self.data_file: - self.data_file.close() - - @lru_cache(maxsize=8) - def __getitem__(self, i) -> torch.Tensor: - if not self.data_file: - self.read_data(self.path) - self.check_index(i) - tensor_size = self.sizes[self.dim_offsets[i] : self.dim_offsets[i + 1]] - a = np.empty(tensor_size, dtype=self.dtype) - self.data_file.seek(self.data_offsets[i] * self.element_size) - self.data_file.readinto(a) - item = torch.from_numpy(a).long() - if self.fix_lua_indexing: - item -= 1 # subtract 1 for 0-based indexing - return item - - def __len__(self): - return self._len - - def num_tokens(self, index): - return self.sizes[index] - - def size(self, index): - return self.sizes[index] - - @staticmethod - def exists(path): - return PathManager.exists(index_file_path(path)) and PathManager.exists( - data_file_path(path) - ) - - @property - def supports_prefetch(self): - return False # avoid prefetching to save memory - - -class IndexedCachedDataset(IndexedDataset): - def __init__(self, path, fix_lua_indexing=False): - super().__init__(path, fix_lua_indexing=fix_lua_indexing) - self.cache = None - self.cache_index = {} - - @property - def supports_prefetch(self): - return True - - def prefetch(self, indices): - if all(i in self.cache_index for i in indices): - return - if not self.data_file: - self.read_data(self.path) - indices = sorted(set(indices)) - total_size = 0 - for i in indices: - total_size += self.data_offsets[i + 1] - self.data_offsets[i] - self.cache = np.empty(total_size, dtype=self.dtype) - ptx = 0 - self.cache_index.clear() - for i in indices: - self.cache_index[i] = ptx - size = self.data_offsets[i + 1] - self.data_offsets[i] - a = self.cache[ptx : ptx + size] - self.data_file.seek(self.data_offsets[i] * self.element_size) - self.data_file.readinto(a) - ptx += size - if self.data_file: - # close and delete data file after prefetch so we can pickle - self.data_file.close() - self.data_file = None - - @lru_cache(maxsize=8) - def __getitem__(self, i): - self.check_index(i) - tensor_size = self.sizes[self.dim_offsets[i] : self.dim_offsets[i + 1]] - a = np.empty(tensor_size, dtype=self.dtype) - ptx = self.cache_index[i] - np.copyto(a, self.cache[ptx : ptx + a.size]) - item = torch.from_numpy(a).long() - if self.fix_lua_indexing: - item -= 1 # subtract 1 for 0-based indexing - return item - - -class IndexedRawTextDataset(FairseqDataset): - """Takes a text file as input and binarizes it in memory at instantiation. - Original lines are also kept in memory""" - - def __init__(self, path, dictionary, append_eos=True, reverse_order=False): - self.tokens_list = [] - self.lines = [] - self.sizes = [] - self.append_eos = append_eos - self.reverse_order = reverse_order - self.read_data(path, dictionary) - self.size = len(self.tokens_list) - - def read_data(self, path, dictionary): - with open(path, "r", encoding="utf-8") as f: - for line in f: - self.lines.append(line.strip("\n")) - tokens = dictionary.encode_line( - line, - add_if_not_exist=False, - append_eos=self.append_eos, - reverse_order=self.reverse_order, - ).long() - self.tokens_list.append(tokens) - self.sizes.append(len(tokens)) - self.sizes = np.array(self.sizes) - - def check_index(self, i): - if i < 0 or i >= self.size: - raise IndexError("index out of range") - - @lru_cache(maxsize=8) - def __getitem__(self, i): - self.check_index(i) - return self.tokens_list[i] - - def get_original_text(self, i): - self.check_index(i) - return self.lines[i] - - def __del__(self): - pass - - def __len__(self): - return self.size - - def num_tokens(self, index): - return self.sizes[index] - - def size(self, index): - return self.sizes[index] - - @staticmethod - def exists(path): - return PathManager.exists(path) - - -class IndexedDatasetBuilder: - element_sizes = { - np.uint8: 1, - np.int8: 1, - np.int16: 2, - np.int32: 4, - np.int64: 8, - np.float64: 4, - np.double: 8, - } - - def __init__(self, out_file, dtype=np.int32): - self.out_file = open(out_file, "wb") - self.dtype = dtype - self.data_offsets = [0] - self.dim_offsets = [0] - self.sizes = [] - self.element_size = self.element_sizes[self.dtype] - - def add_item(self, tensor): - # +1 for Lua compatibility - bytes = self.out_file.write(np.array(tensor.numpy() + 1, dtype=self.dtype)) - self.data_offsets.append(self.data_offsets[-1] + bytes / self.element_size) - for s in tensor.size(): - self.sizes.append(s) - self.dim_offsets.append(self.dim_offsets[-1] + len(tensor.size())) - - def merge_file_(self, another_file): - index = IndexedDataset(another_file) - assert index.dtype == self.dtype - - begin = self.data_offsets[-1] - for offset in index.data_offsets[1:]: - self.data_offsets.append(begin + offset) - self.sizes.extend(index.sizes) - begin = self.dim_offsets[-1] - for dim_offset in index.dim_offsets[1:]: - self.dim_offsets.append(begin + dim_offset) - - with open(data_file_path(another_file), "rb") as f: - while True: - data = f.read(1024) - if data: - self.out_file.write(data) - else: - break - - def finalize(self, index_file): - self.out_file.close() - index = open(index_file, "wb") - index.write(b"TNTIDX\x00\x00") - index.write(struct.pack("<Q", 1)) - index.write( - struct.pack("<QQ", _dtype_header_code(self.dtype), self.element_size) - ) - index.write(struct.pack("<QQ", len(self.data_offsets) - 1, len(self.sizes))) - write_longs(index, self.dim_offsets) - write_longs(index, self.data_offsets) - write_longs(index, self.sizes) - index.close() - - -def _warmup_mmap_file(path): - with open(path, "rb") as stream: - while stream.read(100 * 1024 * 1024): - pass - - -class MMapIndexedDataset(torch.utils.data.Dataset): - class Index: - _HDR_MAGIC = b"MMIDIDX\x00\x00" - - @classmethod - def writer(cls, path, dtype): - class _Writer: - def __enter__(self): - self._file = open(path, "wb") - - self._file.write(cls._HDR_MAGIC) - self._file.write(struct.pack("<Q", 1)) - self._file.write(struct.pack("<B", _dtype_header_code(dtype))) - - return self - - @staticmethod - def _get_pointers(sizes): - dtype_size = dtype().itemsize - address = 0 - pointers = [] - - for size in sizes: - pointers.append(address) - address += size * dtype_size - - return pointers - - def write(self, sizes): - pointers = self._get_pointers(sizes) - - self._file.write(struct.pack("<Q", len(sizes))) - - sizes = np.array(sizes, dtype=np.int32) - self._file.write(sizes.tobytes(order="C")) - del sizes - - pointers = np.array(pointers, dtype=np.int64) - self._file.write(pointers.tobytes(order="C")) - del pointers - - def __exit__(self, exc_type, exc_val, exc_tb): - self._file.close() - - return _Writer() - - def __init__(self, path): - with open(path, "rb") as stream: - magic_test = stream.read(9) - assert self._HDR_MAGIC == magic_test, ( - "Index file doesn't match expected format. " - "Make sure that --dataset-impl is configured properly." - ) - version = struct.unpack("<Q", stream.read(8)) - assert (1,) == version - - (dtype_code,) = struct.unpack("<B", stream.read(1)) - self._dtype = _code_to_dtype[dtype_code] - self._dtype_size = self._dtype().itemsize - - self._len = struct.unpack("<Q", stream.read(8))[0] - offset = stream.tell() - - _warmup_mmap_file(path) - - self._bin_buffer_mmap = np.memmap(path, mode="r", order="C") - self._bin_buffer = memoryview(self._bin_buffer_mmap) - self._sizes = np.frombuffer( - self._bin_buffer, dtype=np.int32, count=self._len, offset=offset - ) - self._pointers = np.frombuffer( - self._bin_buffer, - dtype=np.int64, - count=self._len, - offset=offset + self._sizes.nbytes, - ) - - def __del__(self): - self._bin_buffer_mmap._mmap.close() - del self._bin_buffer_mmap - - @property - def dtype(self): - return self._dtype - - @property - def sizes(self): - return self._sizes - - @lru_cache(maxsize=8) - def __getitem__(self, i): - return self._pointers[i], self._sizes[i] - - def __len__(self): - return self._len - - def __init__(self, path): - super().__init__() - - self._path = None - self._index = None - self._bin_buffer = None - - self._do_init(path) - - def __getstate__(self): - return self._path - - def __setstate__(self, state): - self._do_init(state) - - def _do_init(self, path): - self._path = path - self._index = self.Index(index_file_path(self._path)) - - _warmup_mmap_file(data_file_path(self._path)) - self._bin_buffer_mmap = np.memmap( - data_file_path(self._path), mode="r", order="C" - ) - self._bin_buffer = memoryview(self._bin_buffer_mmap) - - def __del__(self): - self._bin_buffer_mmap._mmap.close() - del self._bin_buffer_mmap - del self._index - - def __len__(self): - return len(self._index) - - @lru_cache(maxsize=8) - def __getitem__(self, i): - ptr, size = self._index[i] - np_array = np.frombuffer( - self._bin_buffer, dtype=self._index.dtype, count=size, offset=ptr - ) - if self._index.dtype != np.int64: - np_array = np_array.astype(np.int64) - - return torch.from_numpy(np_array) - - @property - def sizes(self): - return self._index.sizes - - @property - def supports_prefetch(self): - return False - - @staticmethod - def exists(path): - return PathManager.exists(index_file_path(path)) and PathManager.exists( - data_file_path(path) - ) - - -def get_indexed_dataset_to_local(path) -> str: - local_index_path = PathManager.get_local_path(index_file_path(path)) - local_data_path = PathManager.get_local_path(data_file_path(path)) - - assert local_index_path.endswith(".idx") and local_data_path.endswith(".bin"), ( - "PathManager.get_local_path does not return files with expected patterns: " - f"{local_index_path} and {local_data_path}" - ) - - local_path = local_data_path[:-4] # stripping surfix ".bin" - assert local_path == local_index_path[:-4] # stripping surfix ".idx" - return local_path - - -class MMapIndexedDatasetBuilder: - def __init__(self, out_file, dtype=np.int64): - self._data_file = open(out_file, "wb") - self._dtype = dtype - self._sizes = [] - - def add_item(self, tensor): - np_array = np.array(tensor.numpy(), dtype=self._dtype) - self._data_file.write(np_array.tobytes(order="C")) - self._sizes.append(np_array.size) - - def merge_file_(self, another_file): - # Concatenate index - index = MMapIndexedDataset.Index(index_file_path(another_file)) - assert index.dtype == self._dtype - - for size in index.sizes: - self._sizes.append(size) - - # Concatenate data - with open(data_file_path(another_file), "rb") as f: - shutil.copyfileobj(f, self._data_file) - - def finalize(self, index_file): - self._data_file.close() - - with MMapIndexedDataset.Index.writer(index_file, self._dtype) as index: - index.write(self._sizes) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/iterators.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/iterators.py deleted file mode 100644 index d70f4dd70c32f3cc87563ac1614a0f381a41cd56..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/iterators.py +++ /dev/null @@ -1,820 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import itertools -import logging -import math -import operator -import os -import queue -import time -from threading import Thread - -import numpy as np -import torch -from fairseq.data import data_utils - -logger = logging.getLogger(__name__) - -# Object used by _background_consumer to signal the source is exhausted -# to the main thread. -_sentinel = object() - - -class CountingIterator(object): - """Wrapper around an iterable that maintains the iteration count. - - Args: - iterable (iterable): iterable to wrap - start (int): starting iteration count. Note that this doesn't - actually advance the iterator. - total (int): override the iterator length returned by ``__len``. - This can be used to truncate *iterator*. - - Attributes: - n (int): number of elements consumed from this iterator - """ - - def __init__(self, iterable, start=None, total=None): - self._itr = iter(iterable) - self.n = start or getattr(iterable, "n", 0) - self.total = total if total is not None else self.n + len(iterable) - - def __len__(self): - return self.total - - def __iter__(self): - return self - - def __next__(self): - if not self.has_next(): - raise StopIteration - try: - x = next(self._itr) - except StopIteration: - raise IndexError( - f"Iterator expected to have length {self.total}, " - f"but exhausted at position {self.n}." - ) - self.n += 1 - return x - - def has_next(self): - """Whether the iterator has been exhausted.""" - return self.n < self.total - - def skip(self, n): - """Fast-forward the iterator by skipping n elements.""" - for _ in range(n): - next(self) - return self - - def take(self, n): - """Truncate the iterator to n elements at most.""" - self.total = min(self.total, n) - # Propagate this change to the underlying iterator - if hasattr(self._itr, "take"): - self._itr.take(max(n - self.n, 0)) - return self - - -class EpochBatchIterating(object): - def __len__(self) -> int: - raise NotImplementedError - - @property - def next_epoch_idx(self): - raise NotImplementedError - - def next_epoch_itr( - self, shuffle=True, fix_batches_to_gpus=False, set_dataset_epoch=True - ): - """Return a new iterator over the dataset. - - Args: - shuffle (bool, optional): shuffle batches before returning the - iterator (default: True). - fix_batches_to_gpus (bool, optional): ensure that batches are always - allocated to the same shards across epochs. Requires - that :attr:`dataset` supports prefetching (default: False). - set_dataset_epoch (bool, optional): update the wrapped Dataset with - the new epoch number (default: True). - """ - raise NotImplementedError - - def end_of_epoch(self) -> bool: - """Returns whether the most recent epoch iterator has been exhausted""" - raise NotImplementedError - - @property - def iterations_in_epoch(self) -> int: - """The number of consumed batches in the current epoch.""" - raise NotImplementedError - - def state_dict(self): - """Returns a dictionary containing a whole state of the iterator.""" - raise NotImplementedError - - def load_state_dict(self, state_dict): - """Copies the state of the iterator from the given *state_dict*.""" - raise NotImplementedError - - @property - def first_batch(self): - return "DUMMY" - - -class StreamingEpochBatchIterator(EpochBatchIterating): - """A steaming-style iterator over a :class:`torch.utils.data.IterableDataset`. - - Args: - dataset (~torch.utils.data.Dataset): dataset from which to load the data - max_sentences: batch size - collate_fn (callable): merges a list of samples to form a mini-batch - num_workers (int, optional): how many subprocesses to use for data - loading. 0 means the data will be loaded in the main process - (default: 0). - epoch (int, optional): the epoch to start the iterator from - (default: 1). - buffer_size (int, optional): the number of batches to keep ready in the - queue. Helps speeding up dataloading. When buffer_size is zero, the - default torch.utils.data.DataLoader preloading is used. - timeout (int, optional): if positive, the timeout value for collecting a batch - from workers. Should always be non-negative (default: ``0``). - """ - - def __init__( - self, - dataset, - max_sentences=1, - collate_fn=None, - epoch=1, - num_workers=0, - buffer_size=0, - timeout=0, - ): - assert isinstance(dataset, torch.utils.data.IterableDataset) - self.dataset = dataset - self.max_sentences = max_sentences - self.collate_fn = collate_fn - self.epoch = max(epoch, 1) # we use 1-based indexing for epochs - self.num_workers = num_workers - # This upper limit here is to prevent people from abusing this feature - # in a shared computing environment. - self.buffer_size = min(buffer_size, 20) - self.timeout = timeout - - self._current_epoch_iterator = None - - @property - def next_epoch_idx(self): - """Return the epoch index after *next_epoch_itr* is called.""" - if self._current_epoch_iterator is not None and self.end_of_epoch(): - return self.epoch + 1 - else: - return self.epoch - - def next_epoch_itr( - self, shuffle=True, fix_batches_to_gpus=False, set_dataset_epoch=True - ): - self.epoch = self.next_epoch_idx - if set_dataset_epoch and hasattr(self.dataset, "set_epoch"): - self.dataset.set_epoch(self.epoch) - self._current_epoch_iterator = self._get_iterator_for_epoch(self.epoch, shuffle) - return self._current_epoch_iterator - - def end_of_epoch(self) -> bool: - return not self._current_epoch_iterator.has_next() - - @property - def iterations_in_epoch(self) -> int: - if self._current_epoch_iterator is not None: - return self._current_epoch_iterator.n - return 0 - - def state_dict(self): - return { - "epoch": self.epoch, - } - - def load_state_dict(self, state_dict): - self.epoch = state_dict["epoch"] - - def _get_iterator_for_epoch(self, epoch, shuffle, offset=0): - if self.num_workers > 0: - os.environ["PYTHONWARNINGS"] = "ignore:semaphore_tracker:UserWarning" - - # Create data loader - worker_init_fn = getattr(self.dataset, "worker_init_fn", None) - itr = torch.utils.data.DataLoader( - self.dataset, - batch_size=self.max_sentences, - collate_fn=self.collate_fn, - num_workers=self.num_workers, - timeout=self.timeout, - worker_init_fn=worker_init_fn, - pin_memory=True, - ) - - # Wrap with a BufferedIterator if needed - if self.buffer_size > 0: - itr = BufferedIterator(self.buffer_size, itr) - - # Wrap with CountingIterator - itr = CountingIterator(itr, start=offset) - - return itr - - -class EpochBatchIterator(EpochBatchIterating): - """A multi-epoch iterator over a :class:`torch.utils.data.Dataset`. - - Compared to :class:`torch.utils.data.DataLoader`, this iterator: - - - can be reused across multiple epochs with the :func:`next_epoch_itr` - method (optionally shuffled between epochs) - - can be serialized/deserialized with the :func:`state_dict` and - :func:`load_state_dict` methods - - supports sharding with the *num_shards* and *shard_id* arguments - - Args: - dataset (~torch.utils.data.Dataset): dataset from which to load the data - collate_fn (callable): merges a list of samples to form a mini-batch - batch_sampler (~torch.utils.data.Sampler or a callable): an iterator over batches of - indices, or a callable to create such an iterator (~torch.utils.data.Sampler). - A callable batch_sampler will be called for each epoch to enable per epoch dynamic - batch iterators defined by this callable batch_sampler. - seed (int, optional): seed for random number generator for - reproducibility (default: 1). - num_shards (int, optional): shard the data iterator into N - shards (default: 1). - shard_id (int, optional): which shard of the data iterator to - return (default: 0). - num_workers (int, optional): how many subprocesses to use for data - loading. 0 means the data will be loaded in the main process - (default: 0). - epoch (int, optional): the epoch to start the iterator from - (default: 1). - buffer_size (int, optional): the number of batches to keep ready in the - queue. Helps speeding up dataloading. When buffer_size is zero, the - default torch.utils.data.DataLoader preloading is used. - timeout (int, optional): if positive, the timeout value for collecting a batch - from workers. Should always be non-negative (default: ``0``). - disable_shuffling (bool, optional): force disable shuffling - (default: ``False``). - skip_remainder_batch (bool, optional): if set, discard the last batch in an epoch - for the sake of training stability, as the last batch is usually smaller than - local_batch_size * distributed_word_size (default: ``False``). - grouped_shuffling (bool, optional): enable shuffling batches in groups - of num_shards. Ensures that each GPU receives similar length sequences when - batches are sorted by length. - """ - - def __init__( - self, - dataset, - collate_fn, - batch_sampler, - seed=1, - num_shards=1, - shard_id=0, - num_workers=0, - epoch=1, - buffer_size=0, - timeout=0, - disable_shuffling=False, - skip_remainder_batch=False, - grouped_shuffling=False, - ): - assert isinstance(dataset, torch.utils.data.Dataset) - self.dataset = dataset - self.collate_fn = collate_fn - self.batch_sampler = batch_sampler - self._frozen_batches = ( - tuple(batch_sampler) if not callable(batch_sampler) else None - ) - self.seed = seed - self.num_shards = num_shards - self.shard_id = shard_id - self.num_workers = num_workers - # This upper limit here is to prevent people from abusing this feature - # in a shared computing environment. - self.buffer_size = min(buffer_size, 20) - self.timeout = timeout - self.disable_shuffling = disable_shuffling - self.skip_remainder_batch = skip_remainder_batch - self.grouped_shuffling = grouped_shuffling - - self.epoch = max(epoch, 1) # we use 1-based indexing for epochs - self.shuffle = not disable_shuffling - self._cur_epoch_itr = None - self._next_epoch_itr = None - self._supports_prefetch = getattr(dataset, "supports_prefetch", False) - - @property - def frozen_batches(self): - if self._frozen_batches is None: - self._frozen_batches = tuple(self.batch_sampler(self.dataset, self.epoch)) - return self._frozen_batches - - @property - def first_batch(self): - if len(self.frozen_batches) == 0: - raise Exception( - "The dataset is empty. This could indicate " - "that all elements in the dataset have been skipped. " - "Try increasing the max number of allowed tokens or using " - "a larger dataset." - ) - - if getattr(self.dataset, "supports_fetch_outside_dataloader", True): - return self.collate_fn([self.dataset[i] for i in self.frozen_batches[0]]) - else: - return "DUMMY" - - def __len__(self): - return int(math.ceil(len(self.frozen_batches) / float(self.num_shards))) - - @property - def n(self): - return self.iterations_in_epoch - - @property - def next_epoch_idx(self): - """Return the epoch index after *next_epoch_itr* is called.""" - if self._next_epoch_itr is not None: - return self.epoch - elif self._cur_epoch_itr is not None and self.end_of_epoch(): - return self.epoch + 1 - else: - return self.epoch - - def next_epoch_itr( - self, shuffle=True, fix_batches_to_gpus=False, set_dataset_epoch=True - ): - """Return a new iterator over the dataset. - - Args: - shuffle (bool, optional): shuffle batches before returning the - iterator (default: True). - fix_batches_to_gpus (bool, optional): ensure that batches are always - allocated to the same shards across epochs. Requires - that :attr:`dataset` supports prefetching (default: False). - set_dataset_epoch (bool, optional): update the wrapped Dataset with - the new epoch number (default: True). - """ - if self.disable_shuffling: - shuffle = False - prev_epoch = self.epoch - self.epoch = self.next_epoch_idx - if set_dataset_epoch and hasattr(self.dataset, "set_epoch"): - self.dataset.set_epoch(self.epoch) - if self._next_epoch_itr is not None: - self._cur_epoch_itr = self._next_epoch_itr - self._next_epoch_itr = None - else: - if callable(self.batch_sampler) and prev_epoch != self.epoch: - # reset _frozen_batches to refresh the next epoch - self._frozen_batches = None - self._cur_epoch_itr = self._get_iterator_for_epoch( - self.epoch, - shuffle, - fix_batches_to_gpus=fix_batches_to_gpus, - ) - self.shuffle = shuffle - return self._cur_epoch_itr - - def end_of_epoch(self) -> bool: - """Returns whether the most recent epoch iterator has been exhausted""" - return not self._cur_epoch_itr.has_next() - - @property - def iterations_in_epoch(self): - """The number of consumed batches in the current epoch.""" - if self._cur_epoch_itr is not None: - return self._cur_epoch_itr.n - elif self._next_epoch_itr is not None: - return self._next_epoch_itr.n - return 0 - - def state_dict(self): - """Returns a dictionary containing a whole state of the iterator.""" - if self.end_of_epoch(): - epoch = self.epoch + 1 - iter_in_epoch = 0 - else: - epoch = self.epoch - iter_in_epoch = self.iterations_in_epoch - return { - "version": 2, - "epoch": epoch, - "iterations_in_epoch": iter_in_epoch, - "shuffle": self.shuffle, - } - - def load_state_dict(self, state_dict): - """Copies the state of the iterator from the given *state_dict*.""" - self.epoch = state_dict["epoch"] - itr_pos = state_dict.get("iterations_in_epoch", 0) - version = state_dict.get("version", 1) - if itr_pos > 0: - # fast-forward epoch iterator - self._next_epoch_itr = self._get_iterator_for_epoch( - self.epoch, - shuffle=state_dict.get("shuffle", True), - offset=itr_pos, - ) - if self._next_epoch_itr is None: - if version == 1: - # legacy behavior: we finished the epoch, increment epoch counter - self.epoch += 1 - else: - raise RuntimeError( - "Cannot resume training due to dataloader mismatch, please " - "report this to the fairseq developers. You can relaunch " - "training with `--reset-dataloader` and it should work." - ) - else: - self._next_epoch_itr = None - - def _get_iterator_for_epoch( - self, epoch, shuffle, fix_batches_to_gpus=False, offset=0 - ): - def shuffle_batches(batches, seed): - with data_utils.numpy_seed(seed): - - if self.grouped_shuffling: - grouped_batches = [ - batches[(i * self.num_shards) : ((i + 1) * self.num_shards)] - for i in range((len(batches) // self.num_shards)) - ] - np.random.shuffle(grouped_batches) - batches = list(itertools.chain(*grouped_batches)) - else: - np.random.shuffle(batches) - - return batches - - if self._supports_prefetch: - batches = self.frozen_batches - - if shuffle and not fix_batches_to_gpus: - batches = shuffle_batches(list(batches), self.seed + epoch) - - batches = list( - ShardedIterator(batches, self.num_shards, self.shard_id, fill_value=[]) - ) - self.dataset.prefetch([i for s in batches for i in s]) - - if shuffle and fix_batches_to_gpus: - batches = shuffle_batches(batches, self.seed + epoch + self.shard_id) - else: - if shuffle: - batches = shuffle_batches(list(self.frozen_batches), self.seed + epoch) - else: - batches = self.frozen_batches - batches = list( - ShardedIterator(batches, self.num_shards, self.shard_id, fill_value=[]) - ) - - if offset > 0 and offset >= len(batches): - return None - - if self.num_workers > 0: - os.environ["PYTHONWARNINGS"] = "ignore:semaphore_tracker:UserWarning" - - # Create data loader - itr = torch.utils.data.DataLoader( - self.dataset, - collate_fn=self.collate_fn, - batch_sampler=batches[offset:], - num_workers=self.num_workers, - timeout=self.timeout, - pin_memory=True, - ) - - # Wrap with a BufferedIterator if needed - if self.buffer_size > 0: - itr = BufferedIterator(self.buffer_size, itr) - - # Wrap with CountingIterator - itr = CountingIterator(itr, start=offset) - - if self.skip_remainder_batch: - # TODO: Below is a lazy implementation which discard the final batch regardless - # of whether it is a full batch or not. - total_num_itrs = len(batches) - 1 - itr.take(total_num_itrs) - logger.info(f"skip final residual batch, total_num_itrs = {total_num_itrs}") - - return itr - - -class GroupedIterator(CountingIterator): - """Wrapper around an iterable that returns groups (chunks) of items. - - Args: - iterable (iterable): iterable to wrap - chunk_size (int): size of each chunk - skip_remainder_batch (bool, optional): if set, discard the last grouped batch in - each training epoch, as the last grouped batch is usually smaller than - local_batch_size * distributed_word_size * chunk_size (default: ``False``). - Attributes: - n (int): number of elements consumed from this iterator - """ - - def __init__(self, iterable, chunk_size, skip_remainder_batch=False): - if skip_remainder_batch: - total_num_itrs = int(math.floor(len(iterable) / float(chunk_size))) - logger.info( - f"skip final residual batch, grouped total_num_itrs = {total_num_itrs}" - ) - else: - total_num_itrs = int(math.ceil(len(iterable) / float(chunk_size))) - logger.info(f"grouped total_num_itrs = {total_num_itrs}") - - itr = _chunk_iterator(iterable, chunk_size, skip_remainder_batch) - super().__init__( - itr, - start=int(math.ceil(getattr(iterable, "n", 0) / float(chunk_size))), - total=total_num_itrs, - ) - self.chunk_size = chunk_size - - if skip_remainder_batch: - self.take(total_num_itrs) - # TODO: [Hack] Here the grouped iterator modifies the base iterator size so that - # training can move into the next epoch once the grouped iterator is exhausted. - # Double-check this implementation in case unexpected behavior occurs. - iterable.take(total_num_itrs * chunk_size) - - -def _chunk_iterator(itr, chunk_size, skip_remainder_batch=False): - chunk = [] - for x in itr: - chunk.append(x) - if len(chunk) == chunk_size: - yield chunk - chunk = [] - if not skip_remainder_batch and len(chunk) > 0: - yield chunk - - -class ShardedIterator(CountingIterator): - """A sharded wrapper around an iterable, padded to length. - - Args: - iterable (iterable): iterable to wrap - num_shards (int): number of shards to split the iterable into - shard_id (int): which shard to iterator over - fill_value (Any, optional): padding value when the iterable doesn't - evenly divide *num_shards* (default: None). - - Attributes: - n (int): number of elements consumed from this iterator - """ - - def __init__( - self, iterable, num_shards, shard_id, fill_value=None, skip_remainder_batch=None - ): - """ - Args: - skip_remainder_batch: ignored""" - if shard_id < 0 or shard_id >= num_shards: - raise ValueError("shard_id must be between 0 and num_shards") - sharded_len = int(math.ceil(len(iterable) / float(num_shards))) - itr = map( - operator.itemgetter(1), - itertools.zip_longest( - range(sharded_len), - itertools.islice(iterable, shard_id, len(iterable), num_shards), - fillvalue=fill_value, - ), - ) - super().__init__( - itr, - start=int(math.ceil(getattr(iterable, "n", 0) / float(num_shards))), - total=sharded_len, - ) - - -class BackgroundConsumer(Thread): - def __init__(self, queue, source, max_len, cuda_device): - Thread.__init__(self) - - self._queue = queue - self._source = source - self._max_len = max_len - self.count = 0 - self.cuda_device = cuda_device - - def run(self): - # set_device to avoid creation of GPU0 context when using pin_memory - if self.cuda_device is not None: - torch.cuda.set_device(self.cuda_device) - - try: - for item in self._source: - self._queue.put(item) - - # Stop if we reached the maximum length - self.count += 1 - if self._max_len is not None and self.count >= self._max_len: - break - - # Signal the consumer we are done. - self._queue.put(_sentinel) - except Exception as e: - self._queue.put(e) - - -class BufferedIterator(object): - def __init__(self, size, iterable): - self._queue = queue.Queue(size) - self._iterable = iterable - self._consumer = None - - self.start_time = time.time() - self.warning_time = None - - self.total = len(iterable) - - def _create_consumer(self): - self._consumer = BackgroundConsumer( - self._queue, - self._iterable, - self.total, - torch.cuda.current_device() if torch.cuda.is_available() else None, - ) - self._consumer.daemon = True - self._consumer.start() - - def __iter__(self): - return self - - def __len__(self): - return self.total - - def take(self, n): - self.total = min(self.total, n) - # Propagate this change to the underlying iterator - if hasattr(self._iterable, "take"): - self._iterable.take(n) - return self - - def __next__(self): - # Create consumer if not created yet - if self._consumer is None: - self._create_consumer() - - # Notify the user if there is a data loading bottleneck - if self._queue.qsize() < min(2, max(1, self._queue.maxsize // 2)): - if time.time() - self.start_time > 5 * 60: - if ( - self.warning_time is None - or time.time() - self.warning_time > 15 * 60 - ): - logger.debug( - "Data loading buffer is empty or nearly empty. This may " - "indicate a data loading bottleneck, and increasing the " - "number of workers (--num-workers) may help." - ) - self.warning_time = time.time() - - # Get next example - item = self._queue.get(True) - if isinstance(item, Exception): - raise item - if item is _sentinel: - raise StopIteration() - return item - - -class GroupedEpochBatchIterator(EpochBatchIterator): - """Grouped version of EpochBatchIterator - It takes several samplers from different datasets. - Each epoch shuffle the dataset wise sampler individually with different - random seed. The those sub samplers are combined with into - one big samplers with deterministic permutation to mix batches from - different datasets. It will act like EpochBatchIterator but make sure - 1) data from one data set each time - 2) for different workers, they use the same order to fetch the data - so they will use data from the same dataset everytime - mult_rate is used for update_freq > 1 case where we want to make sure update_freq - mini-batches come from same source - """ - - def __init__( - self, - dataset, - collate_fn, - batch_samplers, - seed=1, - num_shards=1, - shard_id=0, - num_workers=0, - epoch=0, - mult_rate=1, - buffer_size=0, - skip_remainder_batch=False, - ): - super().__init__( - dataset, - collate_fn, - batch_samplers, - seed, - num_shards, - shard_id, - num_workers, - epoch, - buffer_size, - skip_remainder_batch=skip_remainder_batch, - ) - # level 0: sub-samplers 1: batch_idx 2: batches - self._frozen_batches = tuple([tuple(sub_batch) for sub_batch in batch_samplers]) - self.step_size = mult_rate * num_shards - - self.lengths = [ - (len(x) // self.step_size) * self.step_size for x in self.frozen_batches - ] - - def __len__(self): - return sum(self.lengths) - - @property - def first_batch(self): - if len(self.frozen_batches) == 0: - raise Exception( - "The dataset is empty. This could indicate " - "that all elements in the dataset have been skipped. " - "Try increasing the max number of allowed tokens or using " - "a larger dataset." - ) - - if self.dataset.supports_fetch_outside_dataloader: - return self.collate_fn([self.dataset[i] for i in self.frozen_batches[0][0]]) - else: - return "DUMMY" - - def _get_iterator_for_epoch( - self, epoch, shuffle, fix_batches_to_gpus=False, offset=0 - ): - def shuffle_batches(batches, seed): - with data_utils.numpy_seed(seed): - np.random.shuffle(batches) - return batches - - def return_full_batches(batch_sets, seed, shuffle): - if shuffle: - batch_sets = [shuffle_batches(list(x), seed) for x in batch_sets] - - batch_sets = [ - batch_sets[i][: self.lengths[i]] for i in range(len(batch_sets)) - ] - batches = list(itertools.chain.from_iterable(batch_sets)) - - if shuffle: - with data_utils.numpy_seed(seed): - idx = np.random.permutation(len(batches) // self.step_size) - if len(idx) * self.step_size != len(batches): - raise ValueError( - "ERROR: %d %d %d %d" - % (len(idx), self.step_size, len(batches), self.shard_id), - ":".join(["%d" % x for x in self.lengths]), - ) - mini_shards = [ - batches[i * self.step_size : (i + 1) * self.step_size] - for i in idx - ] - batches = list(itertools.chain.from_iterable(mini_shards)) - - return batches - - if self._supports_prefetch: - raise NotImplementedError("To be implemented") - else: - batches = return_full_batches( - self.frozen_batches, self.seed + epoch, shuffle - ) - batches = list( - ShardedIterator(batches, self.num_shards, self.shard_id, fill_value=[]) - ) - - if offset > 0 and offset >= len(batches): - return None - - if self.num_workers > 0: - os.environ["PYTHONWARNINGS"] = "ignore:semaphore_tracker:UserWarning" - - itr = torch.utils.data.DataLoader( - self.dataset, - collate_fn=self.collate_fn, - batch_sampler=batches[offset:], - num_workers=self.num_workers, - ) - if self.buffer_size > 0: - itr = BufferedIterator(self.buffer_size, itr) - - return CountingIterator(itr, start=offset) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/language_pair_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/language_pair_dataset.py deleted file mode 100644 index f050d7370dbbbd48e352c55dc0ca4df2e0077804..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/language_pair_dataset.py +++ /dev/null @@ -1,476 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import logging - -import numpy as np -import torch -from fairseq.data import FairseqDataset, data_utils - -logger = logging.getLogger(__name__) - - -def collate( - samples, - pad_idx, - eos_idx, - left_pad_source=True, - left_pad_target=False, - input_feeding=True, - pad_to_length=None, - pad_to_multiple=1, -): - if len(samples) == 0: - return {} - - def merge(key, left_pad, move_eos_to_beginning=False, pad_to_length=None): - return data_utils.collate_tokens( - [s[key] for s in samples], - pad_idx, - eos_idx, - left_pad, - move_eos_to_beginning, - pad_to_length=pad_to_length, - pad_to_multiple=pad_to_multiple, - ) - - def check_alignment(alignment, src_len, tgt_len): - if alignment is None or len(alignment) == 0: - return False - if ( - alignment[:, 0].max().item() >= src_len - 1 - or alignment[:, 1].max().item() >= tgt_len - 1 - ): - logger.warning("alignment size mismatch found, skipping alignment!") - return False - return True - - def compute_alignment_weights(alignments): - """ - Given a tensor of shape [:, 2] containing the source-target indices - corresponding to the alignments, a weight vector containing the - inverse frequency of each target index is computed. - For e.g. if alignments = [[5, 7], [2, 3], [1, 3], [4, 2]], then - a tensor containing [1., 0.5, 0.5, 1] should be returned (since target - index 3 is repeated twice) - """ - align_tgt = alignments[:, 1] - _, align_tgt_i, align_tgt_c = torch.unique( - align_tgt, return_inverse=True, return_counts=True - ) - align_weights = align_tgt_c[align_tgt_i[np.arange(len(align_tgt))]] - return 1.0 / align_weights.float() - - id = torch.LongTensor([s["id"] for s in samples]) - src_tokens = merge( - "source", - left_pad=left_pad_source, - pad_to_length=pad_to_length["source"] if pad_to_length is not None else None, - ) - # sort by descending source length - src_lengths = torch.LongTensor( - [s["source"].ne(pad_idx).long().sum() for s in samples] - ) - src_lengths, sort_order = src_lengths.sort(descending=True) - id = id.index_select(0, sort_order) - src_tokens = src_tokens.index_select(0, sort_order) - - prev_output_tokens = None - target = None - if samples[0].get("target", None) is not None: - target = merge( - "target", - left_pad=left_pad_target, - pad_to_length=pad_to_length["target"] - if pad_to_length is not None - else None, - ) - target = target.index_select(0, sort_order) - tgt_lengths = torch.LongTensor( - [s["target"].ne(pad_idx).long().sum() for s in samples] - ).index_select(0, sort_order) - ntokens = tgt_lengths.sum().item() - - if samples[0].get("prev_output_tokens", None) is not None: - prev_output_tokens = merge("prev_output_tokens", left_pad=left_pad_target) - elif input_feeding: - # we create a shifted version of targets for feeding the - # previous output token(s) into the next decoder step - prev_output_tokens = merge( - "target", - left_pad=left_pad_target, - move_eos_to_beginning=True, - pad_to_length=pad_to_length["target"] - if pad_to_length is not None - else None, - ) - else: - ntokens = src_lengths.sum().item() - - batch = { - "id": id, - "nsentences": len(samples), - "ntokens": ntokens, - "net_input": { - "src_tokens": src_tokens, - "src_lengths": src_lengths, - }, - "target": target, - } - if prev_output_tokens is not None: - batch["net_input"]["prev_output_tokens"] = prev_output_tokens.index_select( - 0, sort_order - ) - - if samples[0].get("alignment", None) is not None: - bsz, tgt_sz = batch["target"].shape - src_sz = batch["net_input"]["src_tokens"].shape[1] - - offsets = torch.zeros((len(sort_order), 2), dtype=torch.long) - offsets[:, 1] += torch.arange(len(sort_order), dtype=torch.long) * tgt_sz - if left_pad_source: - offsets[:, 0] += src_sz - src_lengths - if left_pad_target: - offsets[:, 1] += tgt_sz - tgt_lengths - - alignments = [ - alignment + offset - for align_idx, offset, src_len, tgt_len in zip( - sort_order, offsets, src_lengths, tgt_lengths - ) - for alignment in [samples[align_idx]["alignment"].view(-1, 2)] - if check_alignment(alignment, src_len, tgt_len) - ] - - if len(alignments) > 0: - alignments = torch.cat(alignments, dim=0) - align_weights = compute_alignment_weights(alignments) - - batch["alignments"] = alignments - batch["align_weights"] = align_weights - - if samples[0].get("constraints", None) is not None: - # Collate the packed constraints across the samples, padding to - # the length of the longest sample. - lens = [sample.get("constraints").size(0) for sample in samples] - max_len = max(lens) - constraints = torch.zeros((len(samples), max(lens))).long() - for i, sample in enumerate(samples): - constraints[i, 0 : lens[i]] = samples[i].get("constraints") - batch["constraints"] = constraints.index_select(0, sort_order) - - return batch - - -class LanguagePairDataset(FairseqDataset): - """ - A pair of torch.utils.data.Datasets. - - Args: - src (torch.utils.data.Dataset): source dataset to wrap - src_sizes (List[int]): source sentence lengths - src_dict (~fairseq.data.Dictionary): source vocabulary - tgt (torch.utils.data.Dataset, optional): target dataset to wrap - tgt_sizes (List[int], optional): target sentence lengths - tgt_dict (~fairseq.data.Dictionary, optional): target vocabulary - left_pad_source (bool, optional): pad source tensors on the left side - (default: True). - left_pad_target (bool, optional): pad target tensors on the left side - (default: False). - shuffle (bool, optional): shuffle dataset elements before batching - (default: True). - input_feeding (bool, optional): create a shifted version of the targets - to be passed into the model for teacher forcing (default: True). - remove_eos_from_source (bool, optional): if set, removes eos from end - of source if it's present (default: False). - append_eos_to_target (bool, optional): if set, appends eos to end of - target if it's absent (default: False). - align_dataset (torch.utils.data.Dataset, optional): dataset - containing alignments. - constraints (Tensor, optional): 2d tensor with a concatenated, zero- - delimited list of constraints for each sentence. - append_bos (bool, optional): if set, appends bos to the beginning of - source/target sentence. - num_buckets (int, optional): if set to a value greater than 0, then - batches will be bucketed into the given number of batch shapes. - src_lang_id (int, optional): source language ID, if set, the collated batch - will contain a field 'src_lang_id' in 'net_input' which indicates the - source language of the samples. - tgt_lang_id (int, optional): target language ID, if set, the collated batch - will contain a field 'tgt_lang_id' which indicates the target language - of the samples. - """ - - def __init__( - self, - src, - src_sizes, - src_dict, - tgt=None, - tgt_sizes=None, - tgt_dict=None, - left_pad_source=True, - left_pad_target=False, - shuffle=True, - input_feeding=True, - remove_eos_from_source=False, - append_eos_to_target=False, - align_dataset=None, - constraints=None, - append_bos=False, - eos=None, - num_buckets=0, - src_lang_id=None, - tgt_lang_id=None, - pad_to_multiple=1, - ): - if tgt_dict is not None: - assert src_dict.pad() == tgt_dict.pad() - assert src_dict.eos() == tgt_dict.eos() - assert src_dict.unk() == tgt_dict.unk() - if tgt is not None: - assert len(src) == len( - tgt - ), "Source and target must contain the same number of examples" - self.src = src - self.tgt = tgt - self.src_sizes = np.array(src_sizes) - self.tgt_sizes = np.array(tgt_sizes) if tgt_sizes is not None else None - self.sizes = ( - np.vstack((self.src_sizes, self.tgt_sizes)).T - if self.tgt_sizes is not None - else self.src_sizes - ) - self.src_dict = src_dict - self.tgt_dict = tgt_dict - self.left_pad_source = left_pad_source - self.left_pad_target = left_pad_target - self.shuffle = shuffle - self.input_feeding = input_feeding - self.remove_eos_from_source = remove_eos_from_source - self.append_eos_to_target = append_eos_to_target - self.align_dataset = align_dataset - if self.align_dataset is not None: - assert ( - self.tgt_sizes is not None - ), "Both source and target needed when alignments are provided" - self.constraints = constraints - self.append_bos = append_bos - self.eos = eos if eos is not None else src_dict.eos() - self.src_lang_id = src_lang_id - self.tgt_lang_id = tgt_lang_id - if num_buckets > 0: - from fairseq.data import BucketPadLengthDataset - - self.src = BucketPadLengthDataset( - self.src, - sizes=self.src_sizes, - num_buckets=num_buckets, - pad_idx=self.src_dict.pad(), - left_pad=self.left_pad_source, - ) - self.src_sizes = self.src.sizes - logger.info("bucketing source lengths: {}".format(list(self.src.buckets))) - if self.tgt is not None: - self.tgt = BucketPadLengthDataset( - self.tgt, - sizes=self.tgt_sizes, - num_buckets=num_buckets, - pad_idx=self.tgt_dict.pad(), - left_pad=self.left_pad_target, - ) - self.tgt_sizes = self.tgt.sizes - logger.info( - "bucketing target lengths: {}".format(list(self.tgt.buckets)) - ) - - # determine bucket sizes using self.num_tokens, which will return - # the padded lengths (thanks to BucketPadLengthDataset) - num_tokens = np.vectorize(self.num_tokens, otypes=[np.compat.long]) - self.bucketed_num_tokens = num_tokens(np.arange(len(self.src))) - self.buckets = [ - (None, num_tokens) for num_tokens in np.unique(self.bucketed_num_tokens) - ] - else: - self.buckets = None - self.pad_to_multiple = pad_to_multiple - - def get_batch_shapes(self): - return self.buckets - - def __getitem__(self, index): - tgt_item = self.tgt[index] if self.tgt is not None else None - src_item = self.src[index] - # Append EOS to end of tgt sentence if it does not have an EOS and remove - # EOS from end of src sentence if it exists. This is useful when we use - # use existing datasets for opposite directions i.e., when we want to - # use tgt_dataset as src_dataset and vice versa - if self.append_eos_to_target: - eos = self.tgt_dict.eos() if self.tgt_dict else self.src_dict.eos() - if self.tgt and self.tgt[index][-1] != eos: - tgt_item = torch.cat([self.tgt[index], torch.LongTensor([eos])]) - - if self.append_bos: - bos = self.tgt_dict.bos() if self.tgt_dict else self.src_dict.bos() - if self.tgt and self.tgt[index][0] != bos: - tgt_item = torch.cat([torch.LongTensor([bos]), self.tgt[index]]) - - bos = self.src_dict.bos() - if self.src[index][0] != bos: - src_item = torch.cat([torch.LongTensor([bos]), self.src[index]]) - - if self.remove_eos_from_source: - eos = self.src_dict.eos() - if self.src[index][-1] == eos: - src_item = self.src[index][:-1] - - example = { - "id": index, - "source": src_item, - "target": tgt_item, - } - if self.align_dataset is not None: - example["alignment"] = self.align_dataset[index] - if self.constraints is not None: - example["constraints"] = self.constraints[index] - return example - - def __len__(self): - return len(self.src) - - def collater(self, samples, pad_to_length=None): - """Merge a list of samples to form a mini-batch. - - Args: - samples (List[dict]): samples to collate - pad_to_length (dict, optional): a dictionary of - {'source': source_pad_to_length, 'target': target_pad_to_length} - to indicate the max length to pad to in source and target respectively. - - Returns: - dict: a mini-batch with the following keys: - - - `id` (LongTensor): example IDs in the original input order - - `ntokens` (int): total number of tokens in the batch - - `net_input` (dict): the input to the Model, containing keys: - - - `src_tokens` (LongTensor): a padded 2D Tensor of tokens in - the source sentence of shape `(bsz, src_len)`. Padding will - appear on the left if *left_pad_source* is ``True``. - - `src_lengths` (LongTensor): 1D Tensor of the unpadded - lengths of each source sentence of shape `(bsz)` - - `prev_output_tokens` (LongTensor): a padded 2D Tensor of - tokens in the target sentence, shifted right by one - position for teacher forcing, of shape `(bsz, tgt_len)`. - This key will not be present if *input_feeding* is - ``False``. Padding will appear on the left if - *left_pad_target* is ``True``. - - `src_lang_id` (LongTensor): a long Tensor which contains source - language IDs of each sample in the batch - - - `target` (LongTensor): a padded 2D Tensor of tokens in the - target sentence of shape `(bsz, tgt_len)`. Padding will appear - on the left if *left_pad_target* is ``True``. - - `tgt_lang_id` (LongTensor): a long Tensor which contains target language - IDs of each sample in the batch - """ - res = collate( - samples, - pad_idx=self.src_dict.pad(), - eos_idx=self.eos, - left_pad_source=self.left_pad_source, - left_pad_target=self.left_pad_target, - input_feeding=self.input_feeding, - pad_to_length=pad_to_length, - pad_to_multiple=self.pad_to_multiple, - ) - if self.src_lang_id is not None or self.tgt_lang_id is not None: - src_tokens = res["net_input"]["src_tokens"] - bsz = src_tokens.size(0) - if self.src_lang_id is not None: - res["net_input"]["src_lang_id"] = ( - torch.LongTensor([[self.src_lang_id]]).expand(bsz, 1).to(src_tokens) - ) - if self.tgt_lang_id is not None: - res["tgt_lang_id"] = ( - torch.LongTensor([[self.tgt_lang_id]]).expand(bsz, 1).to(src_tokens) - ) - return res - - def num_tokens(self, index): - """Return the number of tokens in a sample. This value is used to - enforce ``--max-tokens`` during batching.""" - return max( - self.src_sizes[index], - self.tgt_sizes[index] if self.tgt_sizes is not None else 0, - ) - - def num_tokens_vec(self, indices): - """Return the number of tokens for a set of positions defined by indices. - This value is used to enforce ``--max-tokens`` during batching.""" - sizes = self.src_sizes[indices] - if self.tgt_sizes is not None: - sizes = np.maximum(sizes, self.tgt_sizes[indices]) - return sizes - - def size(self, index): - """Return an example's size as a float or tuple. This value is used when - filtering a dataset with ``--max-positions``.""" - return ( - self.src_sizes[index], - self.tgt_sizes[index] if self.tgt_sizes is not None else 0, - ) - - def ordered_indices(self): - """Return an ordered list of indices. Batches will be constructed based - on this order.""" - if self.shuffle: - indices = np.random.permutation(len(self)).astype(np.int64) - else: - indices = np.arange(len(self), dtype=np.int64) - if self.buckets is None: - # sort by target length, then source length - if self.tgt_sizes is not None: - indices = indices[np.argsort(self.tgt_sizes[indices], kind="mergesort")] - return indices[np.argsort(self.src_sizes[indices], kind="mergesort")] - else: - # sort by bucketed_num_tokens, which is: - # max(padded_src_len, padded_tgt_len) - return indices[ - np.argsort(self.bucketed_num_tokens[indices], kind="mergesort") - ] - - @property - def supports_prefetch(self): - return getattr(self.src, "supports_prefetch", False) and ( - getattr(self.tgt, "supports_prefetch", False) or self.tgt is None - ) - - def prefetch(self, indices): - self.src.prefetch(indices) - if self.tgt is not None: - self.tgt.prefetch(indices) - if self.align_dataset is not None: - self.align_dataset.prefetch(indices) - - def filter_indices_by_size(self, indices, max_sizes): - """Filter a list of sample indices. Remove those that are longer - than specified in max_sizes. - - Args: - indices (np.array): original array of sample indices - max_sizes (int or list[int] or tuple[int]): max sample size, - can be defined separately for src and tgt (then list or tuple) - - Returns: - np.array: filtered sample array - list: list of removed indices - """ - return data_utils.filter_paired_dataset_indices_by_size( - self.src_sizes, - self.tgt_sizes, - indices, - max_sizes, - ) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/legacy/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/legacy/__init__.py deleted file mode 100644 index 04ce8674d4d9346faac45cd85ae6fe7c8d1dd02d..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/legacy/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from .block_pair_dataset import BlockPairDataset -from .masked_lm_dataset import MaskedLMDataset -from .masked_lm_dictionary import BertDictionary, MaskedLMDictionary - -__all__ = [ - "BertDictionary", - "BlockPairDataset", - "MaskedLMDataset", - "MaskedLMDictionary", -] diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/legacy/block_pair_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/legacy/block_pair_dataset.py deleted file mode 100644 index ba069b46052286c531b4f9706d96788732cd2ad2..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/legacy/block_pair_dataset.py +++ /dev/null @@ -1,311 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import math - -import numpy as np -import torch -from fairseq.data import FairseqDataset - - -class BlockPairDataset(FairseqDataset): - """Break a Dataset of tokens into sentence pair blocks for next sentence - prediction as well as masked language model. - - High-level logics are: - 1. break input tensor to tensor blocks - 2. pair the blocks with 50% next sentence and 50% random sentence - 3. return paired blocks as well as related segment labels - - Args: - dataset (~torch.utils.data.Dataset): dataset to break into blocks - sizes: array of sentence lengths - dictionary: dictionary for the task - block_size: maximum block size - break_mode: mode for breaking copurs into block pairs. currently we support - 2 modes - doc: respect document boundaries and each part of the pair should belong to on document - none: don't respect any boundary and cut tokens evenly - short_seq_prob: probability for generating shorter block pairs - doc_break_size: Size for empty line separating documents. Typically 1 if - the sentences have eos, 0 otherwise. - """ - - def __init__( - self, - dataset, - dictionary, - sizes, - block_size, - break_mode="doc", - short_seq_prob=0.1, - doc_break_size=1, - ): - super().__init__() - self.dataset = dataset - self.pad = dictionary.pad() - self.eos = dictionary.eos() - self.cls = dictionary.cls() - self.mask = dictionary.mask() - self.sep = dictionary.sep() - self.break_mode = break_mode - self.dictionary = dictionary - self.short_seq_prob = short_seq_prob - self.block_indices = [] - - assert len(dataset) == len(sizes) - - if break_mode == "doc": - cur_doc = [] - for sent_id, sz in enumerate(sizes): - assert doc_break_size == 0 or sz != 0, ( - "when doc_break_size is non-zero, we expect documents to be" - "separated by a blank line with a single eos." - ) - # empty line as document separator - if sz == doc_break_size: - if len(cur_doc) == 0: - continue - self.block_indices.append(cur_doc) - cur_doc = [] - else: - cur_doc.append(sent_id) - max_num_tokens = block_size - 3 # Account for [CLS], [SEP], [SEP] - self.sent_pairs = [] - self.sizes = [] - for doc_id, doc in enumerate(self.block_indices): - self._generate_sentence_pair(doc, doc_id, max_num_tokens, sizes) - elif break_mode is None or break_mode == "none": - # each block should have half of the block size since we are constructing block pair - sent_length = (block_size - 3) // 2 - total_len = sum(dataset.sizes) - length = math.ceil(total_len / sent_length) - - def block_at(i): - start = i * sent_length - end = min(start + sent_length, total_len) - return (start, end) - - sent_indices = np.array([block_at(i) for i in range(length)]) - sent_sizes = np.array([e - s for s, e in sent_indices]) - dataset_index = self._sent_to_dataset_index(sent_sizes) - - # pair sentences - self._pair_sentences(dataset_index) - else: - raise ValueError("Invalid break_mode: " + break_mode) - - def _pair_sentences(self, dataset_index): - """ - Give a list of evenly cut blocks/sentences, pair these sentences with 50% - consecutive sentences and 50% random sentences. - This is used for none break mode - """ - # pair sentences - for sent_id, sent in enumerate(dataset_index): - next_sent_label = ( - 1 if np.random.rand() > 0.5 and sent_id != len(dataset_index) - 1 else 0 - ) - if next_sent_label: - next_sent = dataset_index[sent_id + 1] - else: - next_sent = dataset_index[ - self._skip_sampling(len(dataset_index), [sent_id, sent_id + 1]) - ] - self.sent_pairs.append((sent, next_sent, next_sent_label)) - - # The current blocks don't include the special tokens but the - # sizes already account for this - self.sizes.append(3 + sent[3] + next_sent[3]) - - def _sent_to_dataset_index(self, sent_sizes): - """ - Build index mapping block indices to the underlying dataset indices - """ - dataset_index = [] - ds_idx, ds_remaining = -1, 0 - for to_consume in sent_sizes: - sent_size = to_consume - if ds_remaining == 0: - ds_idx += 1 - ds_remaining = sent_sizes[ds_idx] - start_ds_idx = ds_idx - start_offset = sent_sizes[ds_idx] - ds_remaining - while to_consume > ds_remaining: - to_consume -= ds_remaining - ds_idx += 1 - ds_remaining = sent_sizes[ds_idx] - ds_remaining -= to_consume - dataset_index.append( - ( - start_ds_idx, # starting index in dataset - start_offset, # starting offset within starting index - ds_idx, # ending index in dataset - sent_size, # sentence length - ) - ) - assert ds_remaining == 0 - assert ds_idx == len(self.dataset) - 1 - return dataset_index - - def _generate_sentence_pair(self, doc, doc_id, max_num_tokens, sizes): - """ - Go through a single document and genrate sentence paris from it - """ - current_chunk = [] - current_length = 0 - curr = 0 - # To provide more randomness, we decrease target seq length for parts of - # samples (10% by default). Note that max_num_tokens is the hard threshold - # for batching and will never be changed. - target_seq_length = max_num_tokens - if np.random.random() < self.short_seq_prob: - target_seq_length = np.random.randint(2, max_num_tokens) - # loop through all sentences in document - while curr < len(doc): - sent_id = doc[curr] - current_chunk.append(sent_id) - current_length = sum(sizes[current_chunk]) - # split chunk and generate pair when exceed target_seq_length or - # finish the loop - if curr == len(doc) - 1 or current_length >= target_seq_length: - # split the chunk into 2 parts - a_end = 1 - if len(current_chunk) > 2: - a_end = np.random.randint(1, len(current_chunk) - 1) - sent_a = current_chunk[:a_end] - len_a = sum(sizes[sent_a]) - # generate next sentence label, note that if there is only 1 sentence - # in current chunk, label is always 0 - next_sent_label = ( - 1 if np.random.rand() > 0.5 and len(current_chunk) != 1 else 0 - ) - if not next_sent_label: - # if next sentence label is 0, sample sent_b from a random doc - target_b_length = target_seq_length - len_a - rand_doc_id = self._skip_sampling(len(self.block_indices), [doc_id]) - random_doc = self.block_indices[rand_doc_id] - random_start = np.random.randint(0, len(random_doc)) - sent_b = [] - len_b = 0 - for j in range(random_start, len(random_doc)): - sent_b.append(random_doc[j]) - len_b = sum(sizes[sent_b]) - if len_b >= target_b_length: - break - # return the second part of the chunk since it's not used - num_unused_segments = len(current_chunk) - a_end - curr -= num_unused_segments - else: - # if next sentence label is 1, use the second part of chunk as sent_B - sent_b = current_chunk[a_end:] - len_b = sum(sizes[sent_b]) - # currently sent_a and sent_B may be longer than max_num_tokens, - # truncate them and return block idx and offsets for them - sent_a, sent_b = self._truncate_sentences( - sent_a, sent_b, max_num_tokens - ) - self.sent_pairs.append((sent_a, sent_b, next_sent_label)) - self.sizes.append(3 + sent_a[3] + sent_b[3]) - current_chunk = [] - curr += 1 - - def _skip_sampling(self, total, skip_ids): - """ - Generate a random integer which is not in skip_ids. Sample range is [0, total) - TODO: ids in skip_ids should be consecutive, we can extend it to more generic version later - """ - rand_id = np.random.randint(total - len(skip_ids)) - return rand_id if rand_id < min(skip_ids) else rand_id + len(skip_ids) - - def _truncate_sentences(self, sent_a, sent_b, max_num_tokens): - """ - Trancate a pair of sentence to limit total length under max_num_tokens - Logics: - 1. Truncate longer sentence - 2. Tokens to be truncated could be at the beginning or the end of the sentnce - Returns: - Truncated sentences represented by dataset idx - """ - len_a, len_b = sum(self.dataset.sizes[sent_a]), sum(self.dataset.sizes[sent_b]) - front_cut_a = front_cut_b = end_cut_a = end_cut_b = 0 - - while True: - total_length = ( - len_a + len_b - front_cut_a - front_cut_b - end_cut_a - end_cut_b - ) - if total_length <= max_num_tokens: - break - - if len_a - front_cut_a - end_cut_a > len_b - front_cut_b - end_cut_b: - if np.random.rand() < 0.5: - front_cut_a += 1 - else: - end_cut_a += 1 - else: - if np.random.rand() < 0.5: - front_cut_b += 1 - else: - end_cut_b += 1 - - # calculate ds indices as well as offsets and return - truncated_sent_a = self._cut_sentence(sent_a, front_cut_a, end_cut_a) - truncated_sent_b = self._cut_sentence(sent_b, front_cut_b, end_cut_b) - return truncated_sent_a, truncated_sent_b - - def _cut_sentence(self, sent, front_cut, end_cut): - """ - Cut a sentence based on the numbers of tokens to be cut from beginning and end - Represent the sentence as dataset idx and return - """ - start_ds_idx, end_ds_idx, offset = sent[0], sent[-1], 0 - target_len = sum(self.dataset.sizes[sent]) - front_cut - end_cut - while front_cut > 0: - if self.dataset.sizes[start_ds_idx] > front_cut: - offset += front_cut - break - else: - front_cut -= self.dataset.sizes[start_ds_idx] - start_ds_idx += 1 - while end_cut > 0: - if self.dataset.sizes[end_ds_idx] > end_cut: - break - else: - end_cut -= self.dataset.sizes[end_ds_idx] - end_ds_idx -= 1 - return start_ds_idx, offset, end_ds_idx, target_len - - def _fetch_block(self, start_ds_idx, offset, end_ds_idx, length): - """ - Fetch a block of tokens based on its dataset idx - """ - buffer = torch.cat( - [self.dataset[idx] for idx in range(start_ds_idx, end_ds_idx + 1)] - ) - s, e = offset, offset + length - return buffer[s:e] - - def __getitem__(self, index): - block1, block2, next_sent_label = self.sent_pairs[index] - block1 = self._fetch_block(*block1) - block2 = self._fetch_block(*block2) - return block1, block2, next_sent_label - - def __len__(self): - return len(self.sizes) - - @property - def supports_prefetch(self): - return getattr(self.dataset, "supports_prefetch", False) - - def prefetch(self, indices): - prefetch_idx = set() - for index in indices: - for block1, block2, _ in [self.sent_pairs[index]]: - for ds_idx in range(block1[0], block1[2] + 1): - prefetch_idx.add(ds_idx) - for ds_idx in range(block2[0], block2[2] + 1): - prefetch_idx.add(ds_idx) - self.dataset.prefetch(prefetch_idx) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/legacy/masked_lm_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/legacy/masked_lm_dataset.py deleted file mode 100644 index dd8ea2c60aff306ab3a756223a298a28d41a4991..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/legacy/masked_lm_dataset.py +++ /dev/null @@ -1,303 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import math -from typing import Dict, List, Tuple - -import numpy as np -import torch -from fairseq.data import Dictionary, FairseqDataset, data_utils -from fairseq.data.concat_dataset import ConcatDataset -from fairseq.data.legacy.block_pair_dataset import BlockPairDataset -from fairseq.data.token_block_dataset import TokenBlockDataset - - -class MaskedLMDataset(FairseqDataset): - """ - A wrapper Dataset for masked language modelling. The dataset - wraps around TokenBlockDataset or BlockedPairDataset and creates a batch - where the input blocks are masked according to the specified masking - probability. Additionally the batch can also contain sentence level targets - if this is specified. - - Args: - dataset: Dataset which generates blocks of data. Only BlockPairDataset - and TokenBlockDataset are supported. - sizes: Sentence lengths - vocab: Dictionary with the vocabulary and special tokens. - pad_idx: Id of padding token in dictionary - mask_idx: Id of mask token in dictionary - classif_token_idx: Id of classification token in dictionary. This is the - token associated with the sentence embedding (Eg: CLS for BERT) - sep_token_idx: Id of separator token in dictionary - (Eg: SEP in BERT) - seed: Seed for random number generator for reproducibility. - shuffle: Shuffle the elements before batching. - has_pairs: Specifies whether the underlying dataset - generates a pair of blocks along with a sentence_target or not. - Setting it to True assumes that the underlying dataset generates a - label for the pair of sentences which is surfaced as - sentence_target. The default value assumes a single block with no - sentence target. - segment_id: An optional segment id for filling in the segment labels - when we are in the single block setting (Eg: XLM). Default is 0. - masking_ratio: specifies what percentage of the blocks should be masked. - masking_prob: specifies the probability of a given token being - replaced with the "MASK" token. - random_token_prob: specifies the probability of a given token being - replaced by a random token from the vocabulary. - """ - - def __init__( - self, - dataset: FairseqDataset, - sizes: np.ndarray, - vocab: Dictionary, - pad_idx: int, - mask_idx: int, - classif_token_idx: int, - sep_token_idx: int, - seed: int = 1, - shuffle: bool = True, - has_pairs: bool = True, - segment_id: int = 0, - masking_ratio: float = 0.15, - masking_prob: float = 0.8, - random_token_prob: float = 0.1, - ): - # Make sure the input datasets are the ones supported - assert ( - isinstance(dataset, TokenBlockDataset) - or isinstance(dataset, BlockPairDataset) - or isinstance(dataset, ConcatDataset) - ), ( - "MaskedLMDataset only wraps TokenBlockDataset or BlockPairDataset or " - "ConcatDataset" - ) - - self.dataset = dataset - self.sizes = np.array(sizes) - self.vocab = vocab - self.pad_idx = pad_idx - self.mask_idx = mask_idx - self.classif_token_idx = classif_token_idx - self.sep_token_idx = sep_token_idx - self.shuffle = shuffle - self.seed = seed - self.has_pairs = has_pairs - self.segment_id = segment_id - self.masking_ratio = masking_ratio - self.masking_prob = masking_prob - self.random_token_prob = random_token_prob - - # If we have only one block then sizes needs to be updated to include - # the classification token - if not has_pairs: - self.sizes = self.sizes + 1 - - def __getitem__(self, index: int): - # if has_pairs, then expect 2 blocks and a sentence target - if self.has_pairs: - (block_one, block_two, sentence_target) = self.dataset[index] - else: - block_one = self.dataset[index] - - return { - "id": index, - "block_one": block_one, - "block_two": block_two if self.has_pairs else None, - "sentence_target": sentence_target if self.has_pairs else None, - } - - def __len__(self): - return len(self.dataset) - - def _mask_block( - self, - sentence: np.ndarray, - mask_idx: int, - pad_idx: int, - dictionary_token_range: Tuple, - ): - """ - Mask tokens for Masked Language Model training - Samples mask_ratio tokens that will be predicted by LM. - - Note:This function may not be efficient enough since we had multiple - conversions between np and torch, we can replace them with torch - operators later. - - Args: - sentence: 1d tensor to be masked - mask_idx: index to use for masking the sentence - pad_idx: index to use for masking the target for tokens we aren't - predicting - dictionary_token_range: range of indices in dictionary which can - be used for random word replacement - (e.g. without special characters) - Return: - masked_sent: masked sentence - target: target with words which we are not predicting replaced - by pad_idx - """ - masked_sent = np.copy(sentence) - sent_length = len(sentence) - mask_num = math.ceil(sent_length * self.masking_ratio) - mask = np.random.choice(sent_length, mask_num, replace=False) - target = np.copy(sentence) - - for i in range(sent_length): - if i in mask: - rand = np.random.random() - - # replace with mask if probability is less than masking_prob - # (Eg: 0.8) - if rand < self.masking_prob: - masked_sent[i] = mask_idx - - # replace with random token if probability is less than - # masking_prob + random_token_prob (Eg: 0.9) - elif rand < (self.masking_prob + self.random_token_prob): - # sample random token from dictionary - masked_sent[i] = np.random.randint( - dictionary_token_range[0], dictionary_token_range[1] - ) - else: - target[i] = pad_idx - - return masked_sent, target - - def _collate(self, samples: List[Dict], pad_idx: int, eos_idx: int): - """ - Does the heavy lifting for creating a batch from the input list of - examples. The logic is as follows: - 1. Mask the input blocks. In case has_pair is True then we have 2 - blocks to mask. - 2. Prepend the first masked block tensor with the special token - used as sentence embedding. Eg: CLS in BERT. This happens - irrespective of the value of has_pair. - 3. If has_pair is True, then append the first masked block with the - special separator token (eg: SEP for BERT) and compute segment - label accordingly. In this case, also append the second masked - block with this special separator token and compute its segment - label. - 4. For the targets tensor, prepend and append with padding index - accordingly. - 5. Concatenate all tensors. - """ - if len(samples) == 0: - return {} - # To ensure determinism, we reset the state of the PRNG after every - # batch based on the seed and the first id of the batch. This ensures - # that across epochs we get the same mask for the same example. This - # is needed for reproducibility and is how BERT does masking - # TODO: Can we add deteminism without this constraint? - with data_utils.numpy_seed(self.seed + samples[0]["id"]): - for s in samples: - - # token range is needed for replacing with random token during - # masking - token_range = (self.vocab.nspecial, len(self.vocab)) - - # mask according to specified probabilities. - masked_blk_one, masked_tgt_one = self._mask_block( - s["block_one"], - self.mask_idx, - self.pad_idx, - token_range, - ) - - tokens = np.concatenate([[self.classif_token_idx], masked_blk_one]) - targets = np.concatenate([[self.pad_idx], masked_tgt_one]) - segments = np.ones(len(tokens)) * self.segment_id - - # if has_pairs is True then we need to add the SEP token to both - # the blocks after masking and re-compute segments based on the new - # lengths. - if self.has_pairs: - tokens_one = np.concatenate([tokens, [self.sep_token_idx]]) - targets_one = np.concatenate([targets, [self.pad_idx]]) - - masked_blk_two, masked_tgt_two = self._mask_block( - s["block_two"], self.mask_idx, self.pad_idx, token_range - ) - tokens_two = np.concatenate([masked_blk_two, [self.sep_token_idx]]) - targets_two = np.concatenate([masked_tgt_two, [self.pad_idx]]) - - # block + 1 sep + 1 special (CLS) - segments_one = np.zeros(len(tokens_one)) - # block + 1 sep - segments_two = np.ones(len(tokens_two)) - - tokens = np.concatenate([tokens_one, tokens_two]) - targets = np.concatenate([targets_one, targets_two]) - segments = np.concatenate([segments_one, segments_two]) - - s["source"] = torch.LongTensor(tokens) - s["segment_labels"] = torch.LongTensor(segments) - s["lm_target"] = torch.LongTensor(targets) - - def merge(key): - return data_utils.collate_tokens( - [s[key] for s in samples], pad_idx, eos_idx, left_pad=False - ) - - return { - "id": torch.LongTensor([s["id"] for s in samples]), - "ntokens": sum(len(s["source"]) for s in samples), - "net_input": { - "src_tokens": merge("source"), - "segment_labels": merge("segment_labels"), - }, - "lm_target": merge("lm_target"), - "sentence_target": torch.LongTensor([s["sentence_target"] for s in samples]) - if self.has_pairs - else None, - "nsentences": len(samples), - } - - def collater(self, samples: List[Dict]): - """Merge a list of samples to form a mini-batch. - - Args: - samples (List[dict]): samples to collate - - Returns: - dict: a mini-batch of data - """ - return self._collate(samples, self.vocab.pad(), self.vocab.eos()) - - def num_tokens(self, index: int): - """ - Return the number of tokens in a sample. This value is used to - enforce max-tokens during batching. - """ - return self.sizes[index] - - def size(self, index: int): - """ - Return an example's size as a float or tuple. This value is used when - filtering a dataset with max-positions. - """ - return self.sizes[index] - - def ordered_indices(self): - """ - Return an ordered list of indices. Batches will be constructed based - on this order. - """ - if self.shuffle: - return np.random.permutation(len(self)) - else: - order = [np.arange(len(self))] - order.append(self.sizes) - return np.lexsort(order) - - @property - def supports_prefetch(self): - return getattr(self.dataset, "supports_prefetch", False) - - def prefetch(self, indices): - self.dataset.prefetch(indices) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/legacy/masked_lm_dictionary.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/legacy/masked_lm_dictionary.py deleted file mode 100644 index dee88f7a3ed72ea465ea4e8ffe7b1c01ff6f57f1..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/legacy/masked_lm_dictionary.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from fairseq.data import Dictionary - - -class MaskedLMDictionary(Dictionary): - """ - Dictionary for Masked Language Modelling tasks. This extends Dictionary by - adding the mask symbol. - """ - - def __init__( - self, - pad="<pad>", - eos="</s>", - unk="<unk>", - mask="<mask>", - ): - super().__init__(pad=pad, eos=eos, unk=unk) - self.mask_word = mask - self.mask_index = self.add_symbol(mask) - self.nspecial = len(self.symbols) - - def mask(self): - """Helper to get index of mask symbol""" - return self.mask_index - - -class BertDictionary(MaskedLMDictionary): - """ - Dictionary for BERT task. This extends MaskedLMDictionary by adding support - for cls and sep symbols. - """ - - def __init__( - self, - pad="<pad>", - eos="</s>", - unk="<unk>", - mask="<mask>", - cls="<cls>", - sep="<sep>", - ): - super().__init__(pad=pad, eos=eos, unk=unk, mask=mask) - self.cls_word = cls - self.sep_word = sep - self.cls_index = self.add_symbol(cls) - self.sep_index = self.add_symbol(sep) - self.nspecial = len(self.symbols) - - def cls(self): - """Helper to get index of cls symbol""" - return self.cls_index - - def sep(self): - """Helper to get index of sep symbol""" - return self.sep_index diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/list_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/list_dataset.py deleted file mode 100644 index 12f00aa43661d6bad701c9e72653ba8779136906..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/list_dataset.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from . import BaseWrapperDataset - - -class ListDataset(BaseWrapperDataset): - def __init__(self, dataset, sizes=None): - super().__init__(dataset) - self._sizes = sizes - - def __iter__(self): - for x in self.dataset: - yield x - - def collater(self, samples): - return samples - - @property - def sizes(self): - return self._sizes - - def num_tokens(self, index): - return self.sizes[index] - - def size(self, index): - return self.sizes[index] - - def set_epoch(self, epoch): - pass diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/lm_context_window_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/lm_context_window_dataset.py deleted file mode 100644 index b1fbf907b4a74a9b8ec9d1ba36367416bcf983b2..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/lm_context_window_dataset.py +++ /dev/null @@ -1,97 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from typing import Dict - -import numpy as np -import torch -from fairseq.data.monolingual_dataset import MonolingualDataset - -from . import FairseqDataset - - -class LMContextWindowDataset(FairseqDataset): - """ - Wraps a MonolingualDataset and provides more context for evaluation. - - Each item in the new dataset will have a maximum size of - ``tokens_per_sample + context_window``. - - Args: - dataset: dataset to wrap - tokens_per_sample (int): the max number of tokens in each dataset item - context_window (int): the number of accumulated tokens to add to each - dataset item - pad_idx (int): padding symbol - """ - - def __init__( - self, - dataset: MonolingualDataset, - tokens_per_sample: int, - context_window: int, - pad_idx: int, - ): - assert context_window > 0 - self.dataset = dataset - self.tokens_per_sample = tokens_per_sample - self.context_window = context_window - self.pad_idx = pad_idx - self.prev_tokens = np.empty([0]) - - def __getitem__(self, index): - return self.dataset[index] - - def __len__(self): - return len(self.dataset) - - def collater(self, samples) -> Dict: - sample = self.dataset.collater(samples) - - pad = self.pad_idx - max_sample_len = self.tokens_per_sample + self.context_window - - bsz, tsz = sample["net_input"]["src_tokens"].shape - start_idxs = [0] * bsz - toks = sample["net_input"]["src_tokens"] - lengths = sample["net_input"]["src_lengths"] - tgt = sample["target"] - new_toks = np.empty([bsz, tsz + self.context_window], dtype=np.int64) - new_tgt = np.full([bsz, tsz + self.context_window], pad, dtype=np.int64) - sample_lens = toks.ne(pad).long().sum(dim=1).cpu() - for i in range(bsz): - sample_len = sample_lens[i] - extra = len(self.prev_tokens) + sample_len - max_sample_len - if extra > 0: - self.prev_tokens = self.prev_tokens[extra:] - pads = np.full(self.context_window - len(self.prev_tokens), pad) - new_toks[i] = np.concatenate([self.prev_tokens, toks[i].numpy(), pads]) - new_tgt[ - i, len(self.prev_tokens) : len(self.prev_tokens) + len(tgt[i]) - ] = tgt[i] - start_idxs[i] = len(self.prev_tokens) - lengths[i] += len(self.prev_tokens) - self.prev_tokens = new_toks[i][new_toks[i] != pad][-self.context_window :] - sample["net_input"]["src_tokens"] = torch.from_numpy(new_toks) - sample["target"] = torch.from_numpy(new_tgt) - sample["start_indices"] = start_idxs - return sample - - def num_tokens(self, index): - return self.dataset.num_tokens(index) - - def size(self, index): - return self.dataset.size(index) - - def ordered_indices(self): - # NOTE we don't shuffle the data to retain access to the previous dataset elements - return np.arange(len(self.dataset)) - - @property - def supports_prefetch(self): - return getattr(self.dataset, "supports_prefetch", False) - - def prefetch(self, indices): - return self.dataset.prefetch(indices) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/lru_cache_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/lru_cache_dataset.py deleted file mode 100644 index a7854ac1701392754ce5795cafe9c634671aebdf..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/lru_cache_dataset.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from functools import lru_cache - -from . import BaseWrapperDataset - - -class LRUCacheDataset(BaseWrapperDataset): - def __init__(self, dataset, token=None): - super().__init__(dataset) - - @lru_cache(maxsize=8) - def __getitem__(self, index): - return self.dataset[index] - - @lru_cache(maxsize=8) - def collater(self, samples): - return self.dataset.collater(samples) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/mask_tokens_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/mask_tokens_dataset.py deleted file mode 100644 index 9123235594c3977994a3ae8a03ab4c9e395cc5de..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/mask_tokens_dataset.py +++ /dev/null @@ -1,220 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from functools import lru_cache - -import numpy as np -import torch -from fairseq.data import Dictionary, data_utils - -from . import BaseWrapperDataset, LRUCacheDataset - - -class MaskTokensDataset(BaseWrapperDataset): - """ - A wrapper Dataset for masked language modeling. - - Input items are masked according to the specified masking probability. - - Args: - dataset: Dataset to wrap. - sizes: Sentence lengths - vocab: Dictionary with the vocabulary and special tokens. - pad_idx: Id of pad token in vocab - mask_idx: Id of mask token in vocab - return_masked_tokens: controls whether to return the non-masked tokens - (the default) or to return a tensor with the original masked token - IDs (and *pad_idx* elsewhere). The latter is useful as targets for - masked LM training. - seed: Seed for random number generator for reproducibility. - mask_prob: probability of replacing a token with *mask_idx*. - leave_unmasked_prob: probability that a masked token is unmasked. - random_token_prob: probability of replacing a masked token with a - random token from the vocabulary. - freq_weighted_replacement: sample random replacement words based on - word frequencies in the vocab. - mask_whole_words: only mask whole words. This should be a byte mask - over vocab indices, indicating whether it is the beginning of a - word. We will extend any mask to encompass the whole word. - bpe: BPE to use for whole-word masking. - mask_multiple_length : repeat each mask index multiple times. Default - value is 1. - mask_stdev : standard deviation of masks distribution in case of - multiple masking. Default value is 0. - """ - - @classmethod - def apply_mask(cls, dataset: torch.utils.data.Dataset, *args, **kwargs): - """Return the source and target datasets for masked LM training.""" - dataset = LRUCacheDataset(dataset) - return ( - LRUCacheDataset(cls(dataset, *args, **kwargs, return_masked_tokens=False)), - LRUCacheDataset(cls(dataset, *args, **kwargs, return_masked_tokens=True)), - ) - - def __init__( - self, - dataset: torch.utils.data.Dataset, - vocab: Dictionary, - pad_idx: int, - mask_idx: int, - return_masked_tokens: bool = False, - seed: int = 1, - mask_prob: float = 0.15, - leave_unmasked_prob: float = 0.1, - random_token_prob: float = 0.1, - freq_weighted_replacement: bool = False, - mask_whole_words: torch.Tensor = None, - mask_multiple_length: int = 1, - mask_stdev: float = 0.0, - ): - assert 0.0 < mask_prob < 1.0 - assert 0.0 <= random_token_prob <= 1.0 - assert 0.0 <= leave_unmasked_prob <= 1.0 - assert random_token_prob + leave_unmasked_prob <= 1.0 - assert mask_multiple_length >= 1 - assert mask_stdev >= 0.0 - - self.dataset = dataset - self.vocab = vocab - self.pad_idx = pad_idx - self.mask_idx = mask_idx - self.return_masked_tokens = return_masked_tokens - self.seed = seed - self.mask_prob = mask_prob - self.leave_unmasked_prob = leave_unmasked_prob - self.random_token_prob = random_token_prob - self.mask_whole_words = mask_whole_words - self.mask_multiple_length = mask_multiple_length - self.mask_stdev = mask_stdev - - if random_token_prob > 0.0: - if freq_weighted_replacement: - weights = np.array(self.vocab.count) - else: - weights = np.ones(len(self.vocab)) - weights[: self.vocab.nspecial] = 0 - self.weights = weights / weights.sum() - - self.epoch = 0 - - @property - def can_reuse_epoch_itr_across_epochs(self): - return True # only the noise changes, not item sizes - - def set_epoch(self, epoch, **unused): - super().set_epoch(epoch) - self.epoch = epoch - - def __getitem__(self, index: int): - return self.__getitem_cached__(self.seed, self.epoch, index) - - @lru_cache(maxsize=8) - def __getitem_cached__(self, seed: int, epoch: int, index: int): - with data_utils.numpy_seed(self.seed, self.epoch, index): - item = self.dataset[index] - sz = len(item) - - assert ( - self.mask_idx not in item - ), "Dataset contains mask_idx (={}), this is not expected!".format( - self.mask_idx, - ) - - if self.mask_whole_words is not None: - word_begins_mask = self.mask_whole_words.gather(0, item) - word_begins_idx = word_begins_mask.nonzero().view(-1) - sz = len(word_begins_idx) - words = np.split(word_begins_mask, word_begins_idx)[1:] - assert len(words) == sz - word_lens = list(map(len, words)) - - # decide elements to mask - mask = np.full(sz, False) - num_mask = int( - # add a random number for probabilistic rounding - self.mask_prob * sz / float(self.mask_multiple_length) - + np.random.rand() - ) - - # multiple masking as described in the vq-wav2vec paper (https://arxiv.org/abs/1910.05453) - mask_idc = np.random.choice(sz, num_mask, replace=False) - if self.mask_stdev > 0.0: - lengths = np.random.normal( - self.mask_multiple_length, self.mask_stdev, size=num_mask - ) - lengths = [max(0, int(round(x))) for x in lengths] - mask_idc = np.asarray( - [ - mask_idc[j] + offset - for j in range(len(mask_idc)) - for offset in range(lengths[j]) - ], - dtype=np.int64, - ) - else: - mask_idc = np.concatenate( - [mask_idc + i for i in range(self.mask_multiple_length)] - ) - mask_idc = mask_idc[mask_idc < len(mask)] - try: - mask[mask_idc] = True - except: # something wrong - print( - "Assigning mask indexes {} to mask {} failed!".format( - mask_idc, mask - ) - ) - raise - - if self.return_masked_tokens: - # exit early if we're just returning the masked tokens - # (i.e., the targets for masked LM training) - if self.mask_whole_words is not None: - mask = np.repeat(mask, word_lens) - new_item = np.full(len(mask), self.pad_idx) - new_item[mask] = item[torch.from_numpy(mask.astype(np.uint8)) == 1] - return torch.from_numpy(new_item) - - # decide unmasking and random replacement - rand_or_unmask_prob = self.random_token_prob + self.leave_unmasked_prob - if rand_or_unmask_prob > 0.0: - rand_or_unmask = mask & (np.random.rand(sz) < rand_or_unmask_prob) - if self.random_token_prob == 0.0: - unmask = rand_or_unmask - rand_mask = None - elif self.leave_unmasked_prob == 0.0: - unmask = None - rand_mask = rand_or_unmask - else: - unmask_prob = self.leave_unmasked_prob / rand_or_unmask_prob - decision = np.random.rand(sz) < unmask_prob - unmask = rand_or_unmask & decision - rand_mask = rand_or_unmask & (~decision) - else: - unmask = rand_mask = None - - if unmask is not None: - mask = mask ^ unmask - - if self.mask_whole_words is not None: - mask = np.repeat(mask, word_lens) - - new_item = np.copy(item) - new_item[mask] = self.mask_idx - if rand_mask is not None: - num_rand = rand_mask.sum() - if num_rand > 0: - if self.mask_whole_words is not None: - rand_mask = np.repeat(rand_mask, word_lens) - num_rand = rand_mask.sum() - - new_item[rand_mask] = np.random.choice( - len(self.vocab), - num_rand, - p=self.weights, - ) - - return torch.from_numpy(new_item) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/monolingual_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/monolingual_dataset.py deleted file mode 100644 index 54fd583b64a3a475324ade6eaaeccf593d747fdc..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/monolingual_dataset.py +++ /dev/null @@ -1,253 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import numpy as np -import torch - -from . import FairseqDataset, data_utils - - -def collate(samples, pad_idx, eos_idx, fixed_pad_length=None, pad_to_bsz=None): - if len(samples) == 0: - return {} - - def merge(key, is_list=False): - if is_list: - res = [] - for i in range(len(samples[0][key])): - res.append( - data_utils.collate_tokens( - [s[key][i] for s in samples], - pad_idx, - eos_idx, - left_pad=False, - pad_to_length=fixed_pad_length, - pad_to_bsz=pad_to_bsz, - ) - ) - return res - else: - return data_utils.collate_tokens( - [s[key] for s in samples], - pad_idx, - eos_idx, - left_pad=False, - pad_to_length=fixed_pad_length, - pad_to_bsz=pad_to_bsz, - ) - - src_tokens = merge("source") - if samples[0]["target"] is not None: - is_target_list = isinstance(samples[0]["target"], list) - target = merge("target", is_target_list) - else: - target = src_tokens - - return { - "id": torch.LongTensor([s["id"] for s in samples]), - "nsentences": len(samples), - "ntokens": sum(len(s["source"]) for s in samples), - "net_input": { - "src_tokens": src_tokens, - "src_lengths": torch.LongTensor([s["source"].numel() for s in samples]), - }, - "target": target, - } - - -class MonolingualDataset(FairseqDataset): - """ - A wrapper around torch.utils.data.Dataset for monolingual data. - - Args: - dataset (torch.utils.data.Dataset): dataset to wrap - sizes (List[int]): sentence lengths - vocab (~fairseq.data.Dictionary): vocabulary - shuffle (bool, optional): shuffle the elements before batching - (default: True). - """ - - def __init__( - self, - dataset, - sizes, - src_vocab, - tgt_vocab=None, - add_eos_for_other_targets=False, - shuffle=False, - targets=None, - add_bos_token=False, - fixed_pad_length=None, - pad_to_bsz=None, - src_lang_idx=None, - tgt_lang_idx=None, - ): - self.dataset = dataset - self.sizes = np.array(sizes) - self.vocab = src_vocab - self.tgt_vocab = tgt_vocab or src_vocab - self.add_eos_for_other_targets = add_eos_for_other_targets - self.shuffle = shuffle - self.add_bos_token = add_bos_token - self.fixed_pad_length = fixed_pad_length - self.pad_to_bsz = pad_to_bsz - self.src_lang_idx = src_lang_idx - self.tgt_lang_idx = tgt_lang_idx - - assert targets is None or all( - t in {"self", "future", "past"} for t in targets - ), "targets must be none or one of 'self', 'future', 'past'" - if targets is not None and len(targets) == 0: - targets = None - self.targets = targets - - def __getitem__(self, index): - if self.targets is not None: - # *future_target* is the original sentence - # *source* is shifted right by 1 (maybe left-padded with eos) - # *past_target* is shifted right by 2 (left-padded as needed) - # - # Left-to-right language models should condition on *source* and - # predict *future_target*. - # Right-to-left language models should condition on *source* and - # predict *past_target*. - source, future_target, past_target = self.dataset[index] - source, target = self._make_source_target( - source, future_target, past_target - ) - else: - source = self.dataset[index] - target = None - source, target = self._maybe_add_bos(source, target) - return {"id": index, "source": source, "target": target} - - def __len__(self): - return len(self.dataset) - - def _make_source_target(self, source, future_target, past_target): - if self.targets is not None: - target = [] - - if ( - self.add_eos_for_other_targets - and (("self" in self.targets) or ("past" in self.targets)) - and source[-1] != self.vocab.eos() - ): - # append eos at the end of source - source = torch.cat([source, source.new([self.vocab.eos()])]) - - if "future" in self.targets: - future_target = torch.cat( - [future_target, future_target.new([self.vocab.pad()])] - ) - if "past" in self.targets: - # first token is before the start of sentence which is only used in "none" break mode when - # add_eos_for_other_targets is False - past_target = torch.cat( - [ - past_target.new([self.vocab.pad()]), - past_target[1:], - source[-2, None], - ] - ) - - for t in self.targets: - if t == "self": - target.append(source) - elif t == "future": - target.append(future_target) - elif t == "past": - target.append(past_target) - else: - raise Exception("invalid target " + t) - - if len(target) == 1: - target = target[0] - else: - target = future_target - - return source, self._filter_vocab(target) - - def _maybe_add_bos(self, source, target): - if self.add_bos_token: - source = torch.cat([source.new([self.vocab.bos()]), source]) - if target is not None: - target = torch.cat([target.new([self.tgt_vocab.bos()]), target]) - return source, target - - def num_tokens_vec(self, indices): - """Return the number of tokens for a set of positions defined by indices. - This value is used to enforce ``--max-tokens`` during batching.""" - return self.sizes[indices] - - def _filter_vocab(self, target): - if len(self.tgt_vocab) != len(self.vocab): - - def _filter(target): - mask = target.ge(len(self.tgt_vocab)) - if mask.any(): - target[mask] = self.tgt_vocab.unk() - return target - - if isinstance(target, list): - return [_filter(t) for t in target] - return _filter(target) - return target - - def collater(self, samples): - """Merge a list of samples to form a mini-batch. - - Args: - samples (List[dict]): samples to collate - - Returns: - dict: a mini-batch with the following keys: - - - `id` (LongTensor): example IDs in the original input order - - `ntokens` (int): total number of tokens in the batch - - `net_input` (dict): the input to the Model, containing keys: - - - `src_tokens` (LongTensor): a padded 2D Tensor of tokens in - the source sentence of shape `(bsz, src_len)`. Padding will - appear on the right. - - - `target` (LongTensor): a padded 2D Tensor of tokens in the - target sentence of shape `(bsz, tgt_len)`. Padding will appear - on the right. - """ - return collate( - samples, - self.vocab.pad(), - self.vocab.eos(), - self.fixed_pad_length, - self.pad_to_bsz, - ) - - def num_tokens(self, index): - """Return the number of tokens in a sample. This value is used to - enforce ``--max-tokens`` during batching.""" - return self.sizes[index] - - def size(self, index): - """Return an example's size as a float or tuple. This value is used when - filtering a dataset with ``--max-positions``.""" - return self.sizes[index] - - def ordered_indices(self): - """Return an ordered list of indices. Batches will be constructed based - on this order.""" - if self.shuffle: - order = [np.random.permutation(len(self))] - else: - order = [np.arange(len(self))] - order.append(self.sizes) - return np.lexsort(order) - - @property - def supports_prefetch(self): - return getattr(self.dataset, "supports_prefetch", False) - - def prefetch(self, indices): - self.dataset.prefetch(indices) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/multi_corpus_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/multi_corpus_dataset.py deleted file mode 100644 index a3f47c720d0215323ffea7eb5cf5fd7766fbefa6..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/multi_corpus_dataset.py +++ /dev/null @@ -1,256 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import logging -import time -from collections import OrderedDict -from typing import Dict, List, Optional - -import numpy as np -from fairseq.data import data_utils - -from . import FairseqDataset - -logger = logging.getLogger(__name__) - - -class MultiCorpusDataset(FairseqDataset): - """ - Stores multiple instances of FairseqDataset together. - Unless batch_sample=True, requires each instance - to be the same dataset, as the collate method needs to work on batches with - samples from each dataset. - - Allows specifying a distribution over the datasets to use. Note that unlike - MultiCorpusSampledDataset, this distribution allows sampling for each item, - rather than on a batch level. Note that datasets with sampling probabilty - of 0 will be skipped. - - Each time ordered_indices() is called, a new sample is generated with - the specified distribution. - - Args: - datasets: a OrderedDict of FairseqDataset instances. - distribution: a List containing the probability of getting an utterance from - corresponding dataset - seed: random seed for sampling the datsets - sort_indices: if true, will sort the ordered indices by size - batch_sample: if true, will ensure each batch is from a single dataset - """ - - def __init__( - self, - datasets: Dict[str, FairseqDataset], - distribution: List[float], - seed: int, - sort_indices: bool = False, - batch_sample: bool = False, - distributed_rank: Optional[int] = None, - ): - super().__init__() - assert isinstance(datasets, OrderedDict) - assert len(datasets) == len(distribution) - assert sum(distribution) == 1 - self.datasets = datasets - self.distribution = distribution - self.seed = seed - self.sort_indices = sort_indices - self.batch_sample = batch_sample - self.distributed_rank = distributed_rank - - # Avoid repeated conversions to list later - self.dataset_list = list(datasets.values()) - self.total_num_instances = 0 - - first_dataset = self.dataset_list[0] - - self.num_instances_per_dataset = [] - self.dataset_offsets = [] - for i, dataset in enumerate(self.dataset_list): - assert isinstance(dataset, FairseqDataset) - assert type(dataset) is type(first_dataset) - self.num_instances_per_dataset.append( - 0 if self.distribution[i] == 0 else len(dataset) - ) - self.dataset_offsets.append(self.total_num_instances) - self.total_num_instances += self.num_instances_per_dataset[i] - - def ordered_indices(self): - start = time.time() - with data_utils.numpy_seed(self.seed, self.epoch): - logger.info( - f"sampling new dataset with seed {self.seed} epoch {self.epoch}" - ) - sampled_indices = [] - num_selected_instances = 0 - - # For each dataset i, sample self.distribution[i] * self.total_num_instances - for i, key in enumerate(self.datasets): - if self.distribution[i] == 0: - # skip dataset if sampling probability is 0 - continue - - if i < len(self.datasets) - 1: - num_instances = int(self.distribution[i] * self.total_num_instances) - high = self.dataset_offsets[i + 1] - else: - num_instances = self.total_num_instances - num_selected_instances - high = self.total_num_instances - - logger.info(f"sampling {num_instances} from {key} dataset") - num_selected_instances += num_instances - - # First, add k copies of the dataset where k = num_instances // len(dataset). - # This ensures an equal distribution of the data points as much as possible. - # For the remaining entries randomly sample them - dataset_size = len(self.datasets[key]) - num_copies = num_instances // dataset_size - dataset_indices = ( - np.random.permutation(high - self.dataset_offsets[i]) - + self.dataset_offsets[i] - )[: num_instances - num_copies * dataset_size] - if num_copies > 0: - sampled_indices += list( - np.concatenate( - ( - np.repeat( - np.arange(self.dataset_offsets[i], high), num_copies - ), - dataset_indices, - ) - ) - ) - else: - sampled_indices += list(dataset_indices) - - assert ( - len(sampled_indices) == self.total_num_instances - ), f"{len(sampled_indices)} vs {self.total_num_instances}" - - np.random.shuffle(sampled_indices) - if self.sort_indices: - sampled_indices.sort(key=lambda i: self.num_tokens(i)) - - logger.info( - "multi_corpus_dataset ordered_indices took {}s".format( - time.time() - start - ) - ) - return np.array(sampled_indices, dtype=np.int64) - - def _map_index(self, index: int): - """ - If dataset A has length N and dataset B has length M - then index 1 maps to index 1 of dataset A, and index N + 1 - maps to index 1 of B. - """ - counter = 0 - for num_instances, key in zip(self.num_instances_per_dataset, self.datasets): - if index < counter + num_instances: - return index - counter, key - counter += num_instances - raise ValueError( - "Invalid index: {}, max: {}".format(index, self.total_num_instances) - ) - - def __len__(self): - """ - Length of this dataset is the sum of individual datasets - """ - return self.total_num_instances - - def __getitem__(self, index): - new_index, key = self._map_index(index) - try: - item = self.datasets[key][new_index] - item["full_id"] = index - return item - except Exception as e: - e.args = (f"Error from {key} dataset", *e.args) - raise - - def collater(self, samples): - """ - If we are doing batch sampling, then pick the right collater to use. - - Otherwise we assume all collaters are the same. - """ - if len(samples) == 0: - return None - if "full_id" in samples[0]: - _, key = self._map_index(samples[0]["full_id"]) - try: - batch = self.datasets[key].collater(samples) - except Exception: - print(f"Collating failed for key {key}", flush=True) - raise - return batch - else: - # Subclasses may override __getitem__ to not specify full_id - return list(self.datasets.values())[0].collater(samples) - - def num_tokens(self, index: int): - index, key = self._map_index(index) - return self.datasets[key].num_tokens(index) - - def size(self, index: int): - index, key = self._map_index(index) - return self.datasets[key].size(index) - - @property - def can_reuse_epoch_itr_across_epochs(self): - return False - - def set_epoch(self, epoch, **unused): - super().set_epoch(epoch) - logger.info(f"setting epoch of multi_corpus_dataset to {epoch}") - self.epoch = epoch - - @property - def supports_prefetch(self): - return False - - @property - def supports_fetch_outside_dataloader(self): - return all( - self.datasets[key].supports_fetch_outside_dataloader - for key in self.datasets - ) - - def batch_by_size( - self, - indices, - max_tokens=None, - max_sentences=None, - required_batch_size_multiple=1, - ): - if not self.batch_sample: - return super().batch_by_size( - indices, max_tokens, max_sentences, required_batch_size_multiple - ) - - dataset_indices = {key: [] for key in self.datasets} - for i in indices: - _, key = self._map_index(i) - dataset_indices[key].append(i) - - batches = [] - for key in dataset_indices: - cur_batches = super().batch_by_size( - np.array(dataset_indices[key], dtype=np.int64), - max_tokens, - max_sentences, - required_batch_size_multiple, - ) - logger.info(f"Created {len(cur_batches)} batches for dataset {key}") - batches += cur_batches - - # If this dataset is used in a distributed training setup, - # then shuffle such that the order is seeded by the distributed rank - # as well - if self.distributed_rank is not None: - with data_utils.numpy_seed(self.seed, self.epoch, self.distributed_rank): - np.random.shuffle(batches) - return batches diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/multi_corpus_sampled_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/multi_corpus_sampled_dataset.py deleted file mode 100644 index e2e9fdf004dd1da519a170a5e8bc225775776f72..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/multi_corpus_sampled_dataset.py +++ /dev/null @@ -1,152 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from collections import OrderedDict -from typing import Callable, Dict, List - -import numpy as np - -from . import FairseqDataset - - -def uniform_sampler(x): - # Sample from uniform distribution - return np.random.choice(x, 1).item() - - -class MultiCorpusSampledDataset(FairseqDataset): - """ - Stores multiple instances of FairseqDataset together and in every iteration - creates a batch by first sampling a dataset according to a specified - probability distribution and then getting instances from that dataset. - - Args: - datasets: an OrderedDict of FairseqDataset instances. - sampling_func: A function for sampling over list of dataset keys. - The default strategy is to sample uniformly. - """ - - def __init__( - self, - datasets: Dict[str, FairseqDataset], - sampling_func: Callable[[List], int] = None, - ): - super().__init__() - assert isinstance(datasets, OrderedDict) - self.datasets = datasets - if sampling_func is None: - sampling_func = uniform_sampler - self.sampling_func = sampling_func - - self.total_num_instances = 0 - for _, dataset in datasets.items(): - assert isinstance(dataset, FairseqDataset) - self.total_num_instances += len(dataset) - - self._ordered_indices = None - - def __len__(self): - """ - Length of this dataset is the sum of individual datasets - """ - return self.total_num_instances - - def ordered_indices(self): - """ - Ordered indices for batching. Here we call the underlying - dataset's ordered_indices() so that we get the same random ordering - as we would have from using the underlying dataset directly. - """ - if self._ordered_indices is None: - self._ordered_indices = OrderedDict( - [ - (key, dataset.ordered_indices()) - for key, dataset in self.datasets.items() - ] - ) - return np.arange(len(self)) - - def _map_index_to_dataset(self, key: int, index: int): - """ - Different underlying datasets have different lengths. In order to ensure - we are not accessing an index outside the range of the current dataset - size, we wrap around. This function should be called after we have - created an ordering for this and all underlying datasets. - """ - assert ( - self._ordered_indices is not None - ), "Must call MultiCorpusSampledDataset.ordered_indices() first" - mapped_index = index % len(self.datasets[key]) - return self._ordered_indices[key][mapped_index] - - def __getitem__(self, index: int): - """ - Get the item associated with index from each underlying dataset. - Since index is in the range of [0, TotalNumInstances], we need to - map the index to the dataset before retrieving the item. - """ - return OrderedDict( - [ - (key, dataset[self._map_index_to_dataset(key, index)]) - for key, dataset in self.datasets.items() - ] - ) - - def collater(self, samples: List[Dict]): - """ - Generate a mini-batch for this dataset. - To convert this into a regular mini-batch we use the following - logic: - 1. Select a dataset using the specified probability distribution. - 2. Call the collater function of the selected dataset. - """ - if len(samples) == 0: - return None - - selected_key = self.sampling_func(list(self.datasets.keys())) - selected_samples = [sample[selected_key] for sample in samples] - return self.datasets[selected_key].collater(selected_samples) - - def num_tokens(self, index: int): - """ - Return an example's length (number of tokens), used for batching. Here - we return the max across all examples at index across all underlying - datasets. - """ - return max( - dataset.num_tokens(self._map_index_to_dataset(key, index)) - for key, dataset in self.datasets.items() - ) - - def size(self, index: int): - """ - Return an example's size as a float or tuple. Here we return the max - across all underlying datasets. This value is used when filtering a - dataset with max-positions. - """ - return max( - dataset.size(self._map_index_to_dataset(key, index)) - for key, dataset in self.datasets.items() - ) - - @property - def supports_prefetch(self): - return all( - getattr(dataset, "supports_prefetch", False) - for dataset in self.datasets.values() - ) - - def prefetch(self, indices): - for key, dataset in self.datasets.items(): - dataset.prefetch( - [self._map_index_to_dataset(key, index) for index in indices] - ) - - @property - def supports_fetch_outside_dataloader(self): - return all( - self.datasets[key].supports_fetch_outside_dataloader - for key in self.datasets - ) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/multilingual/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/multilingual/__init__.py deleted file mode 100644 index 6264236915a7269a4d920ee8213004374dd86a9a..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/multilingual/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/multilingual/multilingual_data_manager.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/multilingual/multilingual_data_manager.py deleted file mode 100644 index 0deb4fccd1a5fcc07756da64dfce772f3fdda757..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/multilingual/multilingual_data_manager.py +++ /dev/null @@ -1,1155 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import itertools -import json -import logging -import math -import os -from argparse import ArgumentError -from collections import OrderedDict, defaultdict - -from fairseq import utils -from fairseq.data import ( - AppendTokenDataset, - ConcatDataset, - Dictionary, - LanguagePairDataset, - PrependTokenDataset, - SampledMultiDataset, - SampledMultiEpochDataset, - StripTokenDataset, - TransformEosLangPairDataset, - TruncateDataset, - data_utils, - indexed_dataset, -) -from fairseq.data.multilingual.multilingual_utils import ( - EncoderLangtok, - LangTokSpec, - LangTokStyle, - augment_dictionary, - get_lang_tok, -) -from fairseq.data.multilingual.sampled_multi_dataset import CollateFormat -from fairseq.file_io import PathManager -from fairseq.utils import FileContentsAction, csv_str_list, eval_str_dict - -logger = logging.getLogger(__name__) - -SRC_DICT_NAME = "src" -TGT_DICT_NAME = "tgt" - - -def _lang_id(dic: Dictionary, lang: str): - """Return language ID index.""" - idx = dic.index(lang) - assert idx != dic.unk_index, "cannot find language ID for lang {}".format(lang) - return idx - - -def load_sampling_weights(from_file): - with open(from_file) as f: - weights = json.load(f) - return weights - - -class MultilingualDatasetManager(object): - def __init__(self, args, lang_pairs, langs, dicts, sampling_method): - super().__init__() - self.args = args - self.seed = args.seed - self.lang_pairs = lang_pairs - self.extra_lang_pairs = ( - list({p for _, v in args.extra_lang_pairs.items() for p in v.split(",")}) - if args.extra_lang_pairs - else [] - ) - self.src_langs = { - p.split("-")[0] for p in args.lang_pairs + self.extra_lang_pairs - } - self.tgt_langs = { - p.split("-")[1] for p in args.lang_pairs + self.extra_lang_pairs - } - self.langs = langs - self.dicts = dicts - self.lang_dict = self.create_lang_dictionary(self.langs) - self.sampling_method = sampling_method - self.sampling_scheduler = None - self._has_sharded_data = False - self._num_shards_dict = {} - self._training_data_sizes = defaultdict(lambda: {}) - - @classmethod - def setup_data_manager(cls, args, lang_pairs, langs, dicts, sampling_method): - return MultilingualDatasetManager( - args, lang_pairs, langs, dicts, sampling_method - ) - - @staticmethod - def add_args(parser): - parser.add_argument( - "data", - help="colon separated path to data directories list, \ - will be iterated upon during epochs in round-robin manner", - action=FileContentsAction, - ) - parser.add_argument( - "--langs", - default=None, - type=csv_str_list, - help="a list of languages comma sperated languages which can appear in lang-pairs; " - "note that the ordering determines language token IDs", - ) - parser.add_argument( - "--lang-dict", - default=None, - type=str, - help="an external file which contains a list of " - "languages which can appear in lang-pairs; " - "note that the ordering determines language token IDs; " - "--langs and --lang-dict are two exclusive options", - ) - parser.add_argument( - "--source-dict", - default=None, - type=str, - help="path to source dictionary; if specified it will override per language dictionary loading", - ) - parser.add_argument( - "--target-dict", - default=None, - type=str, - help="path to target dictionary; if specified it will override per language dictionary loading", - ) - parser.add_argument( - "--lang-tok-style", - default=LangTokStyle.multilingual.value, - type=str, - choices=[LangTokStyle.multilingual.value, LangTokStyle.mbart.value], - help="language token styles", - ) - - parser.add_argument( - "--load-alignments", - action="store_true", - help="load the binarized alignments", - ) - parser.add_argument( - "--left-pad-source", - default="True", - type=str, - metavar="BOOL", - help="pad the source on the left", - ) - parser.add_argument( - "--left-pad-target", - default="False", - type=str, - metavar="BOOL", - help="pad the target on the left", - ) - try: - parser.add_argument( - "--max-source-positions", - default=1024, - type=int, - metavar="N", - help="max number of tokens in the source sequence", - ) - parser.add_argument( - "--max-target-positions", - default=1024, - type=int, - metavar="N", - help="max number of tokens in the target sequence", - ) - except ArgumentError: - # this might have already been defined. Once we transition this to hydra it should be fine to add it here. - pass - parser.add_argument( - "--upsample-primary", - default=1, - type=int, - help="amount to upsample primary dataset", - ) - parser.add_argument( - "--truncate-source", - action="store_true", - default=False, - help="truncate source to max-source-positions", - ) - parser.add_argument( - "--encoder-langtok", - default=None, - type=str, - choices=[EncoderLangtok.src.value, EncoderLangtok.tgt.value], - metavar="SRCTGT", - help="prepend to the beginning of source sentence the source or target " - "language token. (src/tgt)", - ) - parser.add_argument( - "--decoder-langtok", - action="store_true", - help="prepend to the beginning of target sentence the target language token", - ) - parser.add_argument( - "--lang-tok-replacing-bos-eos", action="store_true", default=False - ) - parser.add_argument( - "--enable-lang-ids", - default=False, - action="store_true", - help="whether to include language IDs in samples", - ) - parser.add_argument( - "--enable-reservsed-directions-shared-datasets", - default=False, - action="store_true", - help="whether to allow datasets be used in reversed directions", - ) - - parser.add_argument( - "--extra-data", - help='a dictionary of data name to this path, \ - e.g. {"mined", path_to_mined_data, "denoised": path_to_denoised_data}', - type=lambda uf: eval_str_dict(uf, type=str), - default=None, - ) - parser.add_argument( - "--extra-lang-pairs", - help='a dictionary of data name to the language pairs they serve, \ - e.g. {"mined": comma-separated-lang-pairs, "denoised": comma-separated-lang-pairs}', - type=lambda uf: eval_str_dict(uf, type=str), - default=None, - ) - parser.add_argument( - "--fixed-dictionary", - help="Fixed dictionary to use with model path", - default=None, - type=str, - ) - parser.add_argument( - "--langtoks-specs", - help='a list of comma separated data types that a set of language tokens to be specialized for, \ - e.g. "main,dae,mined". There will be a set of language tokens added to the vocab to \ - distinguish languages in different training data types. If not specified, default language \ - tokens per languages will be added', - default=LangTokSpec.main.value, - type=csv_str_list, - ) - parser.add_argument( - "--langtoks", - help='a dictionary of how to add language tokens, \ - e.g. {"mined": (None, "tgt"), "mono_dae": ("src.dae", "tgt"), "main": \ - ("src", "tgt")}, or {"mined": ("src.mined", "tgt")}', - default=None, - type=lambda uf: eval_str_dict(uf, type=str), - ) - parser.add_argument( - "--sampling-weights-from-file", - help='a file contain a python dictionary of how to sample data sets, \ - e.g. { "main:en_XX-es_XX": 0.2, "mined:en_XX-pt_XX": 0.5, \ - "mono_dae:es_XX-es_XX: 0.3, "main:en_xx-fr_XX": 0.8 }', - default=None, - type=str, - ) - parser.add_argument( - "--sampling-weights", - help='a dictionary of how to sample data sets, \ - e.g. { "main:en_XX-es_XX": 0.2, "mined:en_XX-pt_XX": 0.5, \ - "mono_dae:es_XX-es_XX: 0.3, "main:en_xx-fr_XX": 0.8 }', - default=None, - type=lambda uf: eval_str_dict(uf, type=str), - ) - parser.add_argument( - "--virtual-epoch-size", - default=None, - type=int, - help="virtual epoch size to speed up data loading", - ) - parser.add_argument( - "--virtual-data-size", - default=None, - type=int, - help="virtual data size of the whole joint dataset to speed" - "up data loading and have specific dynamic sampling strategy interval", - ) - - @classmethod - def load_langs(cls, args, **kwargs): - if args.lang_dict and args.langs: - raise ValueError("--langs and --lang-dict can not both be specified") - if args.lang_dict is None and args.langs is None: - logger.warning( - "External language dictionary is not provided; " - "use lang-pairs to infer the set of supported languages. " - "The language ordering is not stable which might cause " - "misalignment in pretraining and finetuning." - ) - # infer from lang_pairs as it is - langs = list( - {x for lang_pair in args.lang_pairs for x in lang_pair.split("-")} - ) - langs = sorted(langs) - logger.info(f"inferred language list: {langs}") - elif args.lang_dict: - with open( - PathManager.get_local_path(args.lang_dict), "r", encoding="utf-8" - ) as f: - langs = [lang.strip() for lang in f.readlines() if lang.strip()] - logger.info( - f"loaded language list from {args.lang_dict} as they are ordered in file" - ) - elif args.langs: - langs = args.langs - logger.info( - f"parsed the language list as they are ordered in the option: {langs}" - ) - return langs - - def has_sharded_data(self, split): - return self._has_sharded_data and split == getattr( - self.args, "train_subset", None - ) - - def _shared_collater(self): - return not (self.args.extra_data and "mono_dae" in self.args.extra_data) and ( - not self.args.lang_tok_replacing_bos_eos - ) - - def estimate_global_pass_epoch(self, epoch): - if self.args.virtual_epoch_size is None or self.args.virtual_data_size is None: - return None - # one epoch more for remaining data in each shard - virtual_epochs_per_shard = math.ceil( - self.args.virtual_data_size / self.args.virtual_epoch_size - ) - # note that fairseq epoch / shard_epoch starts from 1 - shard_epoch = (epoch - 1) // virtual_epochs_per_shard + 1 - return shard_epoch - - @classmethod - def prepare(cls, load_dictionary, args, **kargs): - args.left_pad_source = utils.eval_bool(args.left_pad_source) - args.left_pad_target = utils.eval_bool(args.left_pad_target) - - if not hasattr(args, "shuffle_instance"): - args.shuffle_instance = False - if args.langtoks is None: - args.langtoks = {} - if "main" not in args.langtoks: - src_langtok_spec = args.encoder_langtok if args.encoder_langtok else None - tgt_langtok_spec = "tgt" if args.decoder_langtok else None - args.langtoks["main"] = (src_langtok_spec, tgt_langtok_spec) - - def check_langs(langs, pairs): - messages = [] - for src, tgt in pairs: - if src not in langs or tgt not in langs: - messages.append( - f"language pair {src}-{tgt} contains languages " - "that are not in the language dictionary" - ) - if len(messages) > 0: - raise ValueError(" ".join(messages) + f"; langs: {langs}") - - if args.lang_pairs is None: - raise ValueError( - "--lang-pairs is required. List all the language pairs in the training objective." - ) - if isinstance(args.lang_pairs, str): - args.lang_pairs = args.lang_pairs.split(",") - if args.source_lang is not None or args.target_lang is not None: - training = False - else: - training = True - language_list = cls.load_langs(args, **kargs) - check_langs( - language_list, - ( - [p.split("-") for p in args.lang_pairs] - if training - else [(args.source_lang, args.target_lang)] - ), - ) - - def load_dictionary_and_postproc(path): - d = load_dictionary(path) - augment_dictionary( - dictionary=d, - language_list=language_list, - lang_tok_style=args.lang_tok_style, - langtoks_specs=args.langtoks_specs, - extra_data=args.extra_data, - ) - return d - - dicts = cls.load_all_dictionaries( - args, language_list, load_dictionary_and_postproc, training - ) - return language_list, dicts, training - - @classmethod - def load_all_dictionaries(cls, args, language_list, load_dictionary, training): - dicts = OrderedDict() - if args.source_dict is not None: - dicts[SRC_DICT_NAME] = load_dictionary(args.source_dict) - if args.target_dict is not None: - dicts[TGT_DICT_NAME] = load_dictionary(args.target_dict) - - if training: - extra_lang_pairs = ( - list( - {p for _, v in args.extra_lang_pairs.items() for p in v.split(",")} - ) - if args.extra_lang_pairs - else [] - ) - src_langs_to_load_dicts = sorted( - {p.split("-")[0] for p in (args.lang_pairs + extra_lang_pairs)} - ) - tgt_langs_to_load_dicts = sorted( - {p.split("-")[1] for p in (args.lang_pairs + extra_lang_pairs)} - ) - else: - src_langs_to_load_dicts = [args.source_lang] - tgt_langs_to_load_dicts = [args.target_lang] - - paths = utils.split_paths(args.data) - assert len(paths) > 0 - - def load_dicts(langs_to_load_dicts): - for lang in langs_to_load_dicts: - dicts[lang] = load_dictionary( - os.path.join(paths[0], "dict.{}.txt".format(lang)) - ) - if len(dicts) > 0: - dict0 = next(iter(dicts.values())) - assert dicts[lang].pad() == dict0.pad() - assert dicts[lang].eos() == dict0.eos() - assert dicts[lang].unk() == dict0.unk() - logger.info("[{}] dictionary: {} types".format(lang, len(dicts[lang]))) - - if args.fixed_dictionary is not None: - fixed_dict = load_dictionary(args.fixed_dictionary) - dicts = { - lang: fixed_dict - for lang in src_langs_to_load_dicts + tgt_langs_to_load_dicts - } - else: - if args.source_dict is None: - load_dicts(src_langs_to_load_dicts) - if args.target_dict is None: - load_dicts(tgt_langs_to_load_dicts) - return dicts - - def get_source_dictionary(self, lang): - if self.args.source_dict is not None: - return self.dicts[SRC_DICT_NAME] - else: - return self.dicts[lang] - - def get_target_dictionary(self, lang): - if self.args.target_dict is not None: - return self.dicts[TGT_DICT_NAME] - else: - return self.dicts[lang] - - @classmethod - def create_lang_dictionary(cls, langs): - unk = "<unk>" - # hack to remove symbols other than unk as they are not needed by lang dict - lang_dict = Dictionary(pad=unk, eos=unk, unk=unk, bos=unk) - for lang in langs: - lang_dict.add_symbol(lang) - return lang_dict - - @classmethod - def get_langtok_index(cls, lang_tok, dic): - idx = dic.index(lang_tok) - assert ( - idx != dic.unk_index - ), "cannot find language token {} in the dictionary".format(lang_tok) - return idx - - def get_encoder_langtok(self, src_lang, tgt_lang, spec=None): - if spec is None: - return None - if spec and spec.startswith("src"): - if src_lang is None: - return None - langtok = get_lang_tok( - lang=src_lang, lang_tok_style=self.args.lang_tok_style, spec=spec - ) - else: - if tgt_lang is None: - return None - langtok = get_lang_tok( - lang=tgt_lang, lang_tok_style=self.args.lang_tok_style, spec=spec - ) - return self.get_langtok_index( - langtok, - self.get_source_dictionary(src_lang) - if src_lang - else self.get_target_dictionary(tgt_lang), - ) - - def get_decoder_langtok(self, tgt_lang, spec=None): - if spec is None: - return None - langtok = get_lang_tok( - lang=tgt_lang, lang_tok_style=self.args.lang_tok_style, spec=spec - ) - return self.get_langtok_index(langtok, self.get_target_dictionary(tgt_lang)) - - @classmethod - def load_data(cls, path, vdict, impl): - dataset = data_utils.load_indexed_dataset(path, vdict, impl) - return dataset - - @classmethod - def split_exists(cls, split, src, tgt, lang, data_path, dataset_impl): - filename = os.path.join(data_path, "{}.{}-{}.{}".format(split, src, tgt, lang)) - return indexed_dataset.dataset_exists(filename, impl=dataset_impl) - - def load_lang_dataset( - self, - data_path, - split, - src, - src_dict, - tgt, - tgt_dict, - combine, - dataset_impl, - upsample_primary, - max_source_positions, - prepend_bos=False, - load_alignments=False, - truncate_source=False, - ): - - src_datasets = [] - tgt_datasets = [] - - for k in itertools.count(): - split_k = split + (str(k) if k > 0 else "") - - # infer langcode - if self.split_exists(split_k, src, tgt, src, data_path, dataset_impl): - prefix = os.path.join(data_path, "{}.{}-{}.".format(split_k, src, tgt)) - elif self.split_exists(split_k, tgt, src, src, data_path, dataset_impl): - prefix = os.path.join(data_path, "{}.{}-{}.".format(split_k, tgt, src)) - else: - if k > 0: - break - else: - logger.error( - f"Dataset not found: {data_path}, {split_k}, {src}, {tgt}" - ) - raise FileNotFoundError( - "Dataset not found: {} ({})".format(split, data_path) - ) - - src_dataset = self.load_data(prefix + src, src_dict, dataset_impl) - if truncate_source: - src_dataset = AppendTokenDataset( - TruncateDataset( - StripTokenDataset(src_dataset, src_dict.eos()), - max_source_positions - 1, - ), - src_dict.eos(), - ) - src_datasets.append(src_dataset) - tgt_datasets.append(self.load_data(prefix + tgt, tgt_dict, dataset_impl)) - - logger.info( - "{} {} {}-{} {} examples".format( - data_path, split_k, src, tgt, len(src_datasets[-1]) - ) - ) - - if not combine: - break - - assert len(src_datasets) == len(tgt_datasets) - - if len(src_datasets) == 1: - src_dataset, tgt_dataset = src_datasets[0], tgt_datasets[0] - else: - sample_ratios = [1] * len(src_datasets) - sample_ratios[0] = upsample_primary - src_dataset = ConcatDataset(src_datasets, sample_ratios) - tgt_dataset = ConcatDataset(tgt_datasets, sample_ratios) - - if prepend_bos: - assert hasattr(src_dict, "bos_index") and hasattr(tgt_dict, "bos_index") - src_dataset = PrependTokenDataset(src_dataset, src_dict.bos()) - tgt_dataset = PrependTokenDataset(tgt_dataset, tgt_dict.bos()) - - align_dataset = None - if load_alignments: - align_path = os.path.join( - data_path, "{}.align.{}-{}".format(split, src, tgt) - ) - if indexed_dataset.dataset_exists(align_path, impl=dataset_impl): - align_dataset = data_utils.load_indexed_dataset( - align_path, None, dataset_impl - ) - - return src_dataset, tgt_dataset, align_dataset - - def load_langpair_dataset( - self, - data_path, - split, - src, - src_dict, - tgt, - tgt_dict, - combine, - dataset_impl, - upsample_primary, - left_pad_source, - left_pad_target, - max_source_positions, - max_target_positions, - prepend_bos=False, - load_alignments=False, - truncate_source=False, - src_dataset_transform_func=lambda dataset: dataset, - tgt_dataset_transform_func=lambda dataset: dataset, - src_lang_id=None, - tgt_lang_id=None, - langpairs_sharing_datasets=None, - ): - norm_direction = "-".join(sorted([src, tgt])) - if langpairs_sharing_datasets is not None: - src_dataset = langpairs_sharing_datasets.get( - (data_path, split, norm_direction, src), "NotInCache" - ) - tgt_dataset = langpairs_sharing_datasets.get( - (data_path, split, norm_direction, tgt), "NotInCache" - ) - align_dataset = langpairs_sharing_datasets.get( - (data_path, split, norm_direction, src, tgt), "NotInCache" - ) - - # a hack: any one is not in cache, we need to reload them - if ( - langpairs_sharing_datasets is None - or src_dataset == "NotInCache" - or tgt_dataset == "NotInCache" - or align_dataset == "NotInCache" - or split != getattr(self.args, "train_subset", None) - ): - # source and target datasets can be reused in reversed directions to save memory - # reversed directions of valid and test data will not share source and target datasets - src_dataset, tgt_dataset, align_dataset = self.load_lang_dataset( - data_path, - split, - src, - src_dict, - tgt, - tgt_dict, - combine, - dataset_impl, - upsample_primary, - max_source_positions=max_source_positions, - prepend_bos=prepend_bos, - load_alignments=load_alignments, - truncate_source=truncate_source, - ) - src_dataset = src_dataset_transform_func(src_dataset) - tgt_dataset = tgt_dataset_transform_func(tgt_dataset) - if langpairs_sharing_datasets is not None: - langpairs_sharing_datasets[ - (data_path, split, norm_direction, src) - ] = src_dataset - langpairs_sharing_datasets[ - (data_path, split, norm_direction, tgt) - ] = tgt_dataset - langpairs_sharing_datasets[ - (data_path, split, norm_direction, src, tgt) - ] = align_dataset - if align_dataset is None: - # no align data so flag the reverse direction as well in sharing - langpairs_sharing_datasets[ - (data_path, split, norm_direction, tgt, src) - ] = align_dataset - else: - logger.info( - f"Reusing source and target datasets of [{split}] {tgt}-{src} for reversed direction: " - f"[{split}] {src}-{tgt}: src length={len(src_dataset)}; tgt length={len(tgt_dataset)}" - ) - - return LanguagePairDataset( - src_dataset, - src_dataset.sizes, - src_dict, - tgt_dataset, - tgt_dataset.sizes if tgt_dataset is not None else None, - tgt_dict, - left_pad_source=left_pad_source, - left_pad_target=left_pad_target, - align_dataset=align_dataset, - src_lang_id=src_lang_id, - tgt_lang_id=tgt_lang_id, - ) - - def src_dataset_tranform_func(self, src_lang, tgt_lang, dataset, spec=None): - if self.args.lang_tok_replacing_bos_eos: - # it is handled by self.alter_dataset_langtok - # TODO: Unifiy with alter_dataset_langtok - return dataset - if spec is None: - return dataset - tok = self.get_encoder_langtok(src_lang, tgt_lang, spec) - if tok: - return PrependTokenDataset(dataset, tok) - return dataset - - def tgt_dataset_tranform_func(self, source_lang, target_lang, dataset, spec=None): - if dataset is None: - # note that target dataset can be None during inference time - return None - if self.args.lang_tok_replacing_bos_eos: - # TODO: Unifiy with alter_dataset_langtok - # It is handled by self.alter_dataset_langtok. - # The complication in self.alter_dataset_langtok - # makes a unified framework difficult. - return dataset - # if not self.args.decoder_langtok: - if not spec: - return dataset - tok = self.get_decoder_langtok(target_lang, spec) - if tok: - return PrependTokenDataset(dataset, tok) - return dataset - - def alter_dataset_langtok( - self, - lang_pair_dataset, - src_eos=None, - src_lang=None, - tgt_eos=None, - tgt_lang=None, - src_langtok_spec=None, - tgt_langtok_spec=None, - ): - if src_langtok_spec is None and tgt_langtok_spec is None: - return lang_pair_dataset - - new_src_eos = None - if ( - src_langtok_spec is not None - and src_eos is not None - and (src_lang is not None or tgt_lang is not None) - ): - new_src_eos = self.get_encoder_langtok(src_lang, tgt_lang, src_langtok_spec) - else: - src_eos = None - - new_tgt_bos = None - if tgt_langtok_spec and tgt_eos is not None and tgt_lang is not None: - new_tgt_bos = self.get_decoder_langtok(tgt_lang, tgt_langtok_spec) - else: - tgt_eos = None - - return TransformEosLangPairDataset( - lang_pair_dataset, - src_eos=src_eos, - new_src_eos=new_src_eos, - tgt_bos=tgt_eos, - new_tgt_bos=new_tgt_bos, - ) - - def load_a_dataset( - self, - split, - data_path, - src, - src_dict, - tgt, - tgt_dict, - combine, - prepend_bos=False, - langpairs_sharing_datasets=None, - data_category=None, - **extra_kwargs, - ): - dataset_impl = self.args.dataset_impl - upsample_primary = self.args.upsample_primary - left_pad_source = self.args.left_pad_source - left_pad_target = self.args.left_pad_target - max_source_positions = self.args.max_source_positions - max_target_positions = self.args.max_target_positions - load_alignments = self.args.load_alignments - truncate_source = self.args.truncate_source - src_dataset_transform_func = self.src_dataset_tranform_func - tgt_dataset_transform_func = self.tgt_dataset_tranform_func - enable_lang_ids = self.args.enable_lang_ids - lang_dictionary = self.lang_dict - src_langtok_spec, tgt_langtok_spec = extra_kwargs["langtok_spec"] - - src_langtok = self.get_encoder_langtok(src, tgt, src_langtok_spec) - tgt_langtok = self.get_decoder_langtok(tgt, tgt_langtok_spec) - logger.info( - f"{data_category}:{src}-{tgt} src_langtok: {src_langtok}; tgt_langtok: {tgt_langtok}" - ) - - langpair_ds = self.load_langpair_dataset( - data_path, - split, - src, - src_dict, - tgt, - tgt_dict, - combine, - dataset_impl, - upsample_primary, - left_pad_source, - left_pad_target, - max_source_positions, - max_target_positions, - prepend_bos, - load_alignments, - truncate_source, - src_dataset_transform_func=lambda dataset: src_dataset_transform_func( - src, tgt, dataset, src_langtok_spec - ), - tgt_dataset_transform_func=lambda dataset: tgt_dataset_transform_func( - src, tgt, dataset, tgt_langtok_spec - ), - src_lang_id=_lang_id(lang_dictionary, src) - if enable_lang_ids and lang_dictionary is not None - else None, - tgt_lang_id=_lang_id(lang_dictionary, tgt) - if enable_lang_ids and lang_dictionary is not None - else None, - langpairs_sharing_datasets=langpairs_sharing_datasets, - ) - # TODO: handle modified lang toks for mined data and dae data - if self.args.lang_tok_replacing_bos_eos: - ds = self.alter_dataset_langtok( - langpair_ds, - src_eos=self.get_source_dictionary(src).eos() - if src - else self.get_target_dictionary(tgt).eos(), - src_lang=src, - tgt_eos=self.get_target_dictionary(tgt).eos(), - tgt_lang=tgt, - src_langtok_spec=src_langtok_spec, - tgt_langtok_spec=tgt_langtok_spec, - ) - else: - ds = langpair_ds - return ds - - def load_split_langpair_datasets(self, split, data_param_list): - datasets = [] - langpairs_sharing_datasets = ( - {} if self.args.enable_reservsed_directions_shared_datasets else None - ) - for param in data_param_list: - ds = self.load_a_dataset( - split=split, - langpairs_sharing_datasets=langpairs_sharing_datasets, - **param, - ) - datasets.append(ds) - return datasets - - def get_data_paths_and_lang_pairs(self, split): - datapaths = {"main": self.args.data} - lang_pairs = {"main": self.lang_pairs} - if split == getattr(self.args, "train_subset", None): - # only training data can have extra data and extra language pairs - if self.args.extra_data: - extra_datapaths = self.args.extra_data - datapaths.update(extra_datapaths) - if self.args.extra_lang_pairs: - extra_lang_pairs = { - k: v.split(",") for k, v in self.args.extra_lang_pairs.items() - } - lang_pairs.update(extra_lang_pairs) - return datapaths, lang_pairs - - @classmethod - def get_dataset_key(cls, data_category, src, tgt): - return f"{data_category}:{src}-{tgt}" - - @classmethod - def _get_shard_num_dict(cls, split, paths): - shards = defaultdict(int) - for path in paths: - files = PathManager.ls(path) - directions = set() - for f in files: - if f.startswith(split) and f.endswith(".idx"): - # idx files of the form "{split}.{src}-{tgt}.{lang}.idx" - direction = f.split(".")[-3] - directions.add(direction) - for direction in directions: - shards[direction] += 1 - return shards - - def get_split_num_data_shards(self, split): - if split in self._num_shards_dict: - return self._num_shards_dict[split] - num_shards_dict = {} - data_paths, lang_pairs = self.get_data_paths_and_lang_pairs(split) - - for data_category, paths in data_paths.items(): - if data_category not in lang_pairs: - continue - paths = utils.split_paths(paths) - shards_dict = self._get_shard_num_dict(split, paths) - lang_dirs = [ - lang_pair.split("-") for lang_pair in lang_pairs[data_category] - ] - lang_dirs = [x if len(x) > 1 else (x[0], x[0]) for x in lang_dirs] - for src, tgt in lang_dirs: - key = self.get_dataset_key(data_category, src, tgt) - if "mono_" in data_category: - # monolingual data requires tgt only - assert src is None or src == tgt, ( - f"error: src={src}, " - f"tgt={tgt} for data_category={data_category}" - ) - num_shards_dict[key] = shards_dict[tgt] - else: - if f"{src}-{tgt}" in shards_dict: - num_shards_dict[key] = shards_dict[f"{src}-{tgt}"] - elif f"{tgt}-{src}" in shards_dict: - # follow the fairseq tradition to use reversed direction data if it is not available - num_shards_dict[key] = shards_dict[f"{tgt}-{src}"] - self._num_shards_dict[split] = num_shards_dict - logger.info(f"[{split}] num of shards: {num_shards_dict}") - return num_shards_dict - - @classmethod - def get_shard_id(cls, num_shards, epoch, shard_epoch=None): - shard = epoch if shard_epoch is None else shard_epoch - shard = (shard - 1) % num_shards - return shard - - def get_split_data_path(self, paths, epoch, shard_epoch, num_shards): - path = paths[self.get_shard_id(num_shards, epoch, shard_epoch)] - return path - - def get_split_data_param_list(self, split, epoch, shard_epoch=None): - # TODO: to extend with extra datasets and keys and loop over different shard data paths - param_list = [] - data_paths, lang_pairs = self.get_data_paths_and_lang_pairs(split) - logger.info(f"langtoks settings: {self.args.langtoks}") - split_num_shards_dict = self.get_split_num_data_shards(split) - for data_category, paths in data_paths.items(): - if data_category not in lang_pairs: - continue - paths = utils.split_paths(paths) - assert len(paths) > 0 - if len(paths) > 1: - self._has_sharded_data = True - if split != getattr(self.args, "train_subset", None): - # if not training data set, use the first shard for valid and test - paths = paths[:1] - - if data_category in self.args.langtoks: - lang_tok_spec = self.args.langtoks[data_category] - else: - # default to None - lang_tok_spec = (None, None) - - # infer langcode - lang_dirs = [ - lang_pair.split("-") for lang_pair in lang_pairs[data_category] - ] - lang_dirs = [x if len(x) > 1 else (x[0], x[0]) for x in lang_dirs] - for src, tgt in lang_dirs: - assert src is not None or data_category == "mono_dae", ( - f"error: src={src}, " f"tgt={tgt} for data_category={data_category}" - ) - # logger.info(f"preparing param for {data_category}: {src} - {tgt}") - key = self.get_dataset_key(data_category, src, tgt) - data_path = self.get_split_data_path( - paths, epoch, shard_epoch, split_num_shards_dict[key] - ) - param_list.append( - { - "key": key, - "data_path": data_path, - "split": split, - "src": src, - "src_dict": self.get_source_dictionary(src) - if src and data_category != "mono_dae" - else None, - "tgt": tgt, - "tgt_dict": self.get_target_dictionary(tgt), - "data_category": data_category, - "langtok_spec": lang_tok_spec, - } - ) - return param_list - - def get_train_dataset_sizes( - self, data_param_list, datasets, epoch, shard_epoch=None - ): - num_shards = [ - self.get_split_num_data_shards(param["split"])[param["key"]] - for param in data_param_list - ] - data_sizes = [] - for (key, d), num_shard in zip(datasets, num_shards): - my_data_sizes = self._training_data_sizes[key] - shard_ind = self.get_shard_id(num_shard, epoch, shard_epoch) - if shard_ind not in my_data_sizes: - my_data_sizes[shard_ind] = len(d) - known_size = max(my_data_sizes.values()) - data_sizes.append( - # If we don't know the data size of the shard yet, - # use the the max known data size to approximate. - # Note that we preprocess shards by a designated shard size - # and put any remaining data at the end into the last shard so - # the max shard size approximation is almost correct before loading - # the last shard; after loading the last shard, it will have the - # exact data sizes of the whole data size. - (key, sum(my_data_sizes.get(i, known_size) for i in range(num_shard))) - ) - logger.info( - f"estimated total data sizes of all shards used in sampling ratios: {data_sizes}. " - "Note that if the data a shard has not been loaded yet, use the max known data size to approximate" - ) - return [s for _, s in data_sizes] - - def get_train_sampling_ratios( - self, data_param_list, datasets, epoch=1, shard_epoch=None - ): - data_sizes = self.get_train_dataset_sizes( - data_param_list, datasets, epoch, shard_epoch - ) - sampling_func = self.sampling_method.sampling_method_selector() - sample_ratios = sampling_func(data_sizes) if sampling_func is not None else None - return sample_ratios - - def get_sampling_ratios(self, data_param_list, datasets, epoch, shard_epoch=None): - if self.args.sampling_weights_from_file: - weights = load_sampling_weights(self.args.sampling_weights_from_file) - sample_ratios = [weights[k] for k, _ in datasets] - logger.info( - "| ignoring --sampling-weights when loadding sampling weights " - f"from file {self.args.sampling_weights_from_file}" - ) - elif self.args.sampling_weights: - sample_ratios = [self.args.sampling_weights[k] for k, _ in datasets] - else: - sample_ratios = self.get_train_sampling_ratios( - data_param_list, datasets, epoch, shard_epoch - ) - - if sample_ratios is not None: - logger.info( - "| Upsample ratios: {}".format( - list(zip(map(lambda x: x["key"], data_param_list), sample_ratios)) - ) - ) - assert len(sample_ratios) == len(datasets) - return sample_ratios - - def load_split_datasets( - self, split, training, epoch=1, combine=False, shard_epoch=None, **kwargs - ): - data_param_list = self.get_split_data_param_list( - split, epoch, shard_epoch=shard_epoch - ) - langpairs_sharing_datasets = ( - {} if self.args.enable_reservsed_directions_shared_datasets else None - ) - datasets = [ - ( - param["key"], - self.load_a_dataset( - combine=combine, - langpairs_sharing_datasets=langpairs_sharing_datasets, - **param, - ), - ) - for param in data_param_list - ] - return datasets, data_param_list - - def load_into_concat_dataset(self, split, datasets, data_param_list): - if self.args.lang_tok_replacing_bos_eos: - # TODO: to investigate why TransformEosLangPairDataset doesn't work with ConcatDataset - return SampledMultiDataset( - OrderedDict(datasets), - sampling_ratios=None, - eval_key=None, - collate_format=CollateFormat.single, - virtual_size=None, - split=split, - ) - return ConcatDataset([d for _, d in datasets]) - - def load_sampled_multi_epoch_dataset( - self, split, training, epoch=0, combine=False, shard_epoch=None, **kwargs - ): - datasets, data_param_list = self.load_split_datasets( - split, training, epoch, combine, shard_epoch=shard_epoch, **kwargs - ) - if training and split == getattr(self.args, "train_subset", None): - sample_ratios = self.get_sampling_ratios(data_param_list, datasets, epoch) - return SampledMultiEpochDataset( - OrderedDict(datasets), - epoch=epoch, - shard_epoch=shard_epoch, - # valid and test datasets will be degenerate to concating datasets: - sampling_ratios=sample_ratios, - eval_key=None, - collate_format=CollateFormat.single, - virtual_size=self.args.virtual_data_size, - split=split, - virtual_epoch_size=self.args.virtual_epoch_size, - # if not using lang_tok altering, simplified to use the same collater - shared_collater=self._shared_collater(), - ) - else: - return self.load_into_concat_dataset(split, datasets, data_param_list) - - def load_sampled_multi_dataset( - self, split, training, epoch=0, combine=False, shard_epoch=None, **kwargs - ): - datasets, data_param_list = self.load_split_datasets( - split, training, epoch, combine, shard_epoch=shard_epoch, **kwargs - ) - if training and split == getattr(self.args, "train_subset", None): - sample_ratios = self.get_sampling_ratios(data_param_list, datasets, epoch) - return SampledMultiDataset( - OrderedDict(datasets), - epoch=epoch, - # valid and test datasets will be degerate to concating datasets: - sampling_ratios=sample_ratios, - eval_key=None, - collate_format=CollateFormat.single, - virtual_size=self.args.virtual_data_size, - split=split, - # if not using lang_tok altering, simplified to use the same collater - shared_collater=self._shared_collater(), - ) - else: - return self.load_into_concat_dataset(split, datasets, data_param_list) - - def load_dataset( - self, split, training, epoch=0, combine=False, shard_epoch=None, **kwargs - ): - if self.args.virtual_epoch_size is None: - return self.load_sampled_multi_dataset( - split, training, epoch, combine, shard_epoch, **kwargs - ) - else: - return self.load_sampled_multi_epoch_dataset( - split, training, epoch, combine, shard_epoch, **kwargs - ) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/multilingual/multilingual_utils.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/multilingual/multilingual_utils.py deleted file mode 100644 index b4e0f9828cabfdbe375d05d9152b58bdbd6de7dc..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/multilingual/multilingual_utils.py +++ /dev/null @@ -1,63 +0,0 @@ -from enum import Enum -from typing import Dict, List, Optional, Sequence - -import torch -from fairseq.data import Dictionary - - -class EncoderLangtok(Enum): - """ - Prepend to the beginning of source sentence either the - source or target language token. (src/tgt). - """ - - src = "src" - tgt = "tgt" - - -class LangTokSpec(Enum): - main = "main" - mono_dae = "mono_dae" - - -class LangTokStyle(Enum): - multilingual = "multilingual" - mbart = "mbart" - - -@torch.jit.export -def get_lang_tok( - lang: str, lang_tok_style: str, spec: str = LangTokSpec.main.value -) -> str: - # TOKEN_STYLES can't be defined outside this fn since it needs to be - # TorchScriptable. - TOKEN_STYLES: Dict[str, str] = { - LangTokStyle.mbart.value: "[{}]", - LangTokStyle.multilingual.value: "__{}__", - } - - if spec.endswith("dae"): - lang = f"{lang}_dae" - elif spec.endswith("mined"): - lang = f"{lang}_mined" - style = TOKEN_STYLES[lang_tok_style] - return style.format(lang) - - -def augment_dictionary( - dictionary: Dictionary, - language_list: List[str], - lang_tok_style: str, - langtoks_specs: Sequence[str] = (LangTokSpec.main.value,), - extra_data: Optional[Dict[str, str]] = None, -) -> None: - for spec in langtoks_specs: - for language in language_list: - dictionary.add_symbol( - get_lang_tok(lang=language, lang_tok_style=lang_tok_style, spec=spec) - ) - - if lang_tok_style == LangTokStyle.mbart.value or ( - extra_data is not None and LangTokSpec.mono_dae.value in extra_data - ): - dictionary.add_symbol("<mask>") diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/multilingual/sampled_multi_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/multilingual/sampled_multi_dataset.py deleted file mode 100644 index b935d05a2a452a8eae4cf9fed33e208ba9cc0200..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/multilingual/sampled_multi_dataset.py +++ /dev/null @@ -1,467 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import datetime -import hashlib -import logging -import time -from bisect import bisect_right -from collections import OrderedDict, defaultdict -from enum import Enum -from typing import List - -import numpy as np -import torch -from fairseq.data import FairseqDataset, data_utils -from fairseq.distributed import utils as distributed_utils - - -def get_time_gap(s, e): - return ( - datetime.datetime.fromtimestamp(e) - datetime.datetime.fromtimestamp(s) - ).__str__() - - -logger = logging.getLogger(__name__) - - -def default_virtual_size_func(datasets, ratios, max_scale_up=1.5): - sizes = [len(d) for d in datasets] - if ratios is None: - return sum(sizes) - largest_idx = np.argmax(sizes) - largest_r = ratios[largest_idx] - largest_s = sizes[largest_idx] - # set virtual sizes relative to the largest dataset - virtual_sizes = [(r / largest_r) * largest_s for r in ratios] - vsize = sum(virtual_sizes) - max_size = sum(sizes) * max_scale_up - return int(vsize if vsize < max_size else max_size) - - -class CollateFormat(Enum): - single = 1 - ordered_dict = 2 - - -class SampledMultiDataset(FairseqDataset): - """Samples from multiple sub-datasets according to given sampling ratios. - Args: - datasets ( - List[~torch.utils.data.Dataset] - or OrderedDict[str, ~torch.utils.data.Dataset] - ): datasets - sampling_ratios (List[float]): list of probability of each dataset to be sampled - (default: None, which corresponds to concatenating all dataset together). - seed (int): RNG seed to use (default: 2). - epoch (int): starting epoch number (default: 1). - eval_key (str, optional): a key used at evaluation time that causes - this instance to pass-through batches from *datasets[eval_key]*. - collate_format (CollateFormat): collater output format, either CollateFormat.ordered_dict or - CollateFormat.single (default: CollateFormat.single) where CollateFormat.single configures - the collater to output batches of data mixed from all sub-datasets, - and CollateFormat.ordered_dict configures the collater to output a dictionary of batches indexed by keys - of sub-datasets. - Note that not all sub-datasets will present in a single batch in both formats. - virtual_size (int, or callable): the expected virtual size of the dataset (default: default_virtual_size_func). - split (str): the split of the data, e.g. 'train', 'valid' or 'test'. - shared_collater (bool): whether or not to all sub-datasets have the same collater. - shuffle (bool): whether or not to shuffle data (default: True). - """ - - def __init__( - self, - datasets, - sampling_ratios=None, - seed=2, - epoch=1, - eval_key=None, - collate_format=CollateFormat.single, - virtual_size=default_virtual_size_func, - split="", - shared_collater=False, - shuffle=True, - ): - super().__init__() - self.shared_collater = shared_collater - self.shuffle = shuffle - - if isinstance(datasets, OrderedDict): - self.keys = list(datasets.keys()) - datasets = list(datasets.values()) - elif isinstance(datasets, List): - self.keys = list(range(len(datasets))) - else: - raise AssertionError() - self.datasets = datasets - self.split = split - - self.eval_key = eval_key - if self.eval_key is not None: - self.collate_format = CollateFormat.single - else: - self.collate_format = collate_format - - self.seed = seed - self._cur_epoch = None - - self.cumulated_sizes = None - # self.datasets[k][self._cur_indices[i]] is the data item i in this sampled dataset - # namely, data item i is sampled from the kth sub-dataset self.datasets[k] - # where self.cumulated_sizes[k-1] <= i < self.cumulated_sizes[k] - self._cur_indices = None - - self._sizes = None - self.virtual_size_per_dataset = None - # caching properties - self._reset_cached_properties() - self.setup_sampling(sampling_ratios, virtual_size) - self.set_epoch(epoch) - - def _clean_if_not_none(self, var_list): - for v in var_list: - if v is not None: - del v - - def _reset_cached_properties(self): - self._clean_if_not_none([self._sizes, self._cur_indices]) - self._sizes = None - self._cur_indices = None - - def setup_sampling(self, sample_ratios, virtual_size): - sizes = [len(d) for d in self.datasets] - if sample_ratios is None: - # default back to concating datasets - self.sample_ratios = None - self.virtual_size = sum(sizes) - else: - if not isinstance(sample_ratios, np.ndarray): - sample_ratios = np.array(sample_ratios) - self.sample_ratios = sample_ratios - virtual_size = ( - default_virtual_size_func if virtual_size is None else virtual_size - ) - self.virtual_size = ( - virtual_size(self.datasets, self.sample_ratios) - if callable(virtual_size) - else virtual_size - ) - - def adjust_sampling(self, epoch, sampling_ratios, virtual_size): - if sampling_ratios is not None: - sampling_ratios = self._sync_sample_ratios(sampling_ratios) - self.setup_sampling(sampling_ratios, virtual_size) - - def _sync_sample_ratios(self, ratios): - # in case the ratios are not precisely the same across processes - # also to ensure every procresses update the ratios in the same pace - ratios = torch.DoubleTensor(ratios) - if torch.distributed.is_initialized(): - if torch.cuda.is_available(): - distributed_utils.all_reduce( - ratios.cuda(), group=distributed_utils.get_data_parallel_group() - ) - else: - distributed_utils.all_reduce( - ratios, group=distributed_utils.get_data_parallel_group() - ) - ret = ratios.cpu() - ret = ret.numpy() - return ret - - def random_choice_in_dataset(self, rng, dataset, choice_size): - if hasattr(dataset, "random_choice_in_dataset"): - return dataset.random_choice_in_dataset(rng, choice_size) - dataset_size = len(dataset) - return rng.choice( - dataset_size, choice_size, replace=(choice_size > dataset_size) - ) - - def get_virtual_indices(self, rng, datasets, sample_ratios, virtual_size): - def get_counts(sample_ratios): - counts = np.array([virtual_size * r for r in sample_ratios], dtype=np.int64) - diff = virtual_size - counts.sum() - assert diff >= 0 - # due to round-offs, the size might not match the desired sizes - if diff > 0: - dataset_indices = rng.choice( - len(sample_ratios), size=diff, p=sample_ratios - ) - for i in dataset_indices: - counts[i] += 1 - return counts - - def get_in_dataset_indices(datasets, sizes, sample_ratios): - counts = get_counts(sample_ratios) - # uniformally sample desired counts for each dataset - # if the desired counts are large, sample with replacement: - indices = [ - self.random_choice_in_dataset(rng, d, c) - for c, d in zip(counts, datasets) - ] - return indices - - sizes = [len(d) for d in datasets] - if sample_ratios is None: - # default back to concating datasets - in_dataset_indices = [list(range(s)) for s in sizes] - virtual_sizes_per_dataset = sizes - else: - ratios = sample_ratios / sample_ratios.sum() - in_dataset_indices = get_in_dataset_indices(datasets, sizes, ratios) - virtual_sizes_per_dataset = [len(d) for d in in_dataset_indices] - virtual_sizes_per_dataset = np.array(virtual_sizes_per_dataset, np.int64) - cumulative_sizes = np.cumsum(virtual_sizes_per_dataset) - assert sum(virtual_sizes_per_dataset) == virtual_size - assert cumulative_sizes[-1] == virtual_size - if virtual_size < sum(sizes): - logger.warning( - f"virtual data size ({virtual_size}) is less than real data size ({sum(sizes)})." - " If virtual size << real data size, there could be data coverage issue." - ) - in_dataset_indices = np.hstack(in_dataset_indices) - return in_dataset_indices, cumulative_sizes, virtual_sizes_per_dataset - - def _get_dataset_and_index(self, index): - i = bisect_right(self.cumulated_sizes, index) - return i, self._cur_indices[index] - - def __getitem__(self, index): - # self.__getitem__(index) returns self.datasets[k][self._cur_indices[index]] - # where k satisfies self.cumulated_sizes[k - 1] <= k < self.cumulated_sizes[k] - ds_idx, ds_sample_idx = self._get_dataset_and_index(index) - ret = (ds_idx, self.datasets[ds_idx][ds_sample_idx]) - return ret - - def num_tokens(self, index): - return self.sizes[index].max() - - def num_tokens_vec(self, indices): - sizes_vec = self.sizes[np.array(indices)] - # max across all dimensions but first one - return np.amax(sizes_vec, axis=tuple(range(1, len(sizes_vec.shape)))) - - def size(self, index): - return self.sizes[index] - - def __len__(self): - return self.virtual_size - - def collater(self, samples, **extra_args): - """Merge a list of samples to form a mini-batch.""" - if len(samples) == 0: - return None - if self.collate_format == "ordered_dict": - collect_samples = [[] for _ in range(len(self.datasets))] - for (i, sample) in samples: - collect_samples[i].append(sample) - batch = OrderedDict( - [ - (self.keys[i], dataset.collater(collect_samples[i])) - for i, (key, dataset) in enumerate(zip(self.keys, self.datasets)) - if len(collect_samples[i]) > 0 - ] - ) - elif self.shared_collater: - batch = self.datasets[0].collater([s for _, s in samples]) - else: - samples_dict = defaultdict(list) - pad_to_length = ( - defaultdict(int) - if "pad_to_length" not in extra_args - else extra_args["pad_to_length"] - ) - for ds_idx, s in samples: - pad_to_length["source"] = max( - pad_to_length["source"], s["source"].size(0) - ) - if s["target"] is not None: - pad_to_length["target"] = max( - pad_to_length["target"], s["target"].size(0) - ) - samples_dict[ds_idx].append(s) - batches = [ - self.datasets[i].collater(samples_dict[i], pad_to_length=pad_to_length) - for i in range(len(self.datasets)) - if len(samples_dict[i]) > 0 - ] - - def straight_data(tensors): - batch = torch.cat(tensors, dim=0) - return batch - - src_lengths = straight_data( - [b["net_input"]["src_lengths"] for b in batches] - ) - src_lengths, sort_order = src_lengths.sort(descending=True) - - def straight_order(tensors): - batch = straight_data(tensors) - return batch.index_select(0, sort_order) - - batch = { - "id": straight_order([b["id"] for b in batches]), - "nsentences": sum(b["nsentences"] for b in batches), - "ntokens": sum(b["ntokens"] for b in batches), - "net_input": { - "src_tokens": straight_order( - [b["net_input"]["src_tokens"] for b in batches] - ), - "src_lengths": src_lengths, - }, - "target": straight_order([b["target"] for b in batches]) - if batches[0]["target"] is not None - else None, - } - if "prev_output_tokens" in batches[0]["net_input"]: - batch["net_input"]["prev_output_tokens"] = straight_order( - [b["net_input"]["prev_output_tokens"] for b in batches] - ) - if "src_lang_id" in batches[0]["net_input"]: - batch["net_input"]["src_lang_id"] = straight_order( - [b["net_input"]["src_lang_id"] for b in batches] - ) - if "tgt_lang_id" in batches[0]: - batch["tgt_lang_id"] = straight_order( - [b["tgt_lang_id"] for b in batches] - ) - return batch - - @property - def sizes(self): - if self._sizes is not None: - return self._sizes - start_time = time.time() - in_sub_dataset_indices = [ - self._cur_indices[ - 0 if i == 0 else self.cumulated_sizes[i - 1] : self.cumulated_sizes[i] - ] - for i in range(len(self.datasets)) - ] - sub_dataset_sizes = [ - d.sizes[indices] - for d, indices in zip(self.datasets, in_sub_dataset_indices) - ] - self._sizes = np.vstack(sub_dataset_sizes) - logger.info(f"sizes() calling time: {get_time_gap(start_time, time.time())}") - return self._sizes - - def ordered_indices(self): - if self.shuffle: - indices = np.random.permutation(len(self)) - else: - indices = np.arange(len(self)) - - sizes = self.sizes - tgt_sizes = sizes[:, 1] if len(sizes.shape) > 0 and sizes.shape[1] > 1 else None - src_sizes = ( - sizes[:, 0] if len(sizes.shape) > 0 and sizes.shape[1] > 1 else sizes - ) - - # sort by target length, then source length - if tgt_sizes is not None: - indices = indices[np.argsort(tgt_sizes[indices], kind="mergesort")] - sort_indices = indices[np.argsort(src_sizes[indices], kind="mergesort")] - return sort_indices - - def prefetch(self, indices): - prefetch_indices = [[] for _ in range(len(self.datasets))] - for i in indices: - ds_idx, ds_sample_idx = self._get_dataset_and_index(i) - prefetch_indices[ds_idx].append(ds_sample_idx) - for i in range(len(prefetch_indices)): - self.datasets[i].prefetch(prefetch_indices[i]) - - @property - def can_reuse_epoch_itr_across_epochs(self): - return False - - def set_epoch(self, epoch): - super().set_epoch(epoch) - if epoch == self._cur_epoch: - # re-enter so return - return - for d in self.datasets: - if hasattr(d, "set_epoch"): - d.set_epoch(epoch) - self._cur_epoch = epoch - self._establish_virtual_datasets() - - def _establish_virtual_datasets(self): - if self.sample_ratios is None and self._cur_indices is not None: - # not a samping dataset, no need to resample if indices are already established - return - self._reset_cached_properties() - - start_time = time.time() - # Generate a weighted sample of indices as a function of the - # random seed and the current epoch. - rng = np.random.RandomState( - [ - int( - hashlib.sha1( - str(self.__class__.__name__).encode("utf-8") - ).hexdigest(), - 16, - ) - % (2**32), - self.seed % (2**32), # global seed - self._cur_epoch, # epoch index, - ] - ) - self._clean_if_not_none( - [self.cumulated_sizes, self.virtual_size_per_dataset, self._sizes] - ) - self._sizes = None - - indices, cumulated_sizes, virtual_size_per_dataset = self.get_virtual_indices( - rng, self.datasets, self.sample_ratios, self.virtual_size - ) - self._cur_indices = indices - self.cumulated_sizes = cumulated_sizes - self.virtual_size_per_dataset = virtual_size_per_dataset - - raw_sizes = [len(d) for d in self.datasets] - sampled_sizes = self.virtual_size_per_dataset - logger.info( - f"[{self.split}] Raw sizes: {str(dict(zip(self.keys, raw_sizes)))}; " - f"raw total size: {sum(raw_sizes)}" - ) - logger.info( - f"[{self.split}] Resampled sizes: {str(dict(zip(self.keys, sampled_sizes)))}; " - f"resampled total size: {sum(sampled_sizes)}" - ) - if self.sample_ratios is not None: - logger.info( - f"[{self.split}] Upsampling ratios: {str(dict(zip(self.keys, self.sample_ratios)))}" - ) - else: - logger.info(f"[{self.split}] A concat dataset") - logger.info( - f"[{self.split}] virtual dataset established time: {get_time_gap(start_time, time.time())}" - ) - - def filter_indices_by_size(self, indices, max_sizes): - """Filter a list of sample indices. Remove those that are longer - than specified in max_sizes. - - Args: - indices (np.array): original array of sample indices - max_sizes (int or list[int] or tuple[int]): max sample size, - can be defined separately for src and tgt (then list or tuple) - - Returns: - np.array: filtered sample array - list: list of removed indices - """ - sizes = self.sizes - tgt_sizes = sizes[:, 1] if len(sizes.shape) > 0 and sizes.shape[1] > 1 else None - src_sizes = ( - sizes[:, 0] if len(sizes.shape) > 0 and sizes.shape[1] > 1 else sizes - ) - - return data_utils.filter_paired_dataset_indices_by_size( - src_sizes, tgt_sizes, indices, max_sizes - ) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/multilingual/sampled_multi_epoch_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/multilingual/sampled_multi_epoch_dataset.py deleted file mode 100644 index f69c4c410c219a023a46f5f8d1a47a8f28abcf4a..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/multilingual/sampled_multi_epoch_dataset.py +++ /dev/null @@ -1,198 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import hashlib -import logging -import math - -import numpy as np -from fairseq.data import SampledMultiDataset - -from .sampled_multi_dataset import CollateFormat, default_virtual_size_func - -logger = logging.getLogger(__name__) - - -class SampledMultiEpochDataset(SampledMultiDataset): - """Samples from multiple sub-datasets according to sampling ratios - using virtual epoch sizes to speed up dataloading. - Args: - datasets ( - List[~torch.utils.data.Dataset] - or OrderedDict[str, ~torch.utils.data.Dataset] - ): datasets - sampling_ratios (List[float]): list of probability of each dataset to be sampled - (default: None, which corresponds to concating all dataset together). - seed (int): RNG seed to use (default: 2). - epoch (int): starting epoch number (default: 1). - eval_key (str, optional): a key used at evaluation time that causes - this instance to pass-through batches from *datasets[eval_key]*. - collate_format (CollateFormat): collater output format, either CollateFormat.ordered_dict or - CollateFormat.single (default: CollateFormat.single) where CollateFormat.single configures - the collater to output batches of data mixed from all sub-datasets, - and CollateFormat.ordered_dict configures the collater to output a dictionary of batches indexed by keys - of sub-datasets. - Note that not all sub-datasets will present in a single batch in both formats. - virtual_size (int, or callable): the expected virtual size of the dataset (default: default_virtual_size_func). - split (str): the split of the data, e.g. 'train', 'valid' or 'test'. - virtual_epoch_size (int): virtual epoch size, the dataset will go through the data by - this virtual epoch size one by one to speed up data loading, e.g. indicing and filtering - can be performed whenever a virtual epoch is loaded without waiting for the whole dataset to be loaded. - shared_collater (bool): whether or not to all sub-datasets have the same collater. - shard_epoch (int): the real epoch number for shard selection. - shuffle (bool): whether or not to shuffle data (default: True). - """ - - def __init__( - self, - datasets, - sampling_ratios=None, - seed=2, - epoch=1, - eval_key=None, - collate_format=CollateFormat.single, - virtual_size=default_virtual_size_func, - split="", - virtual_epoch_size=None, - shared_collater=False, - shard_epoch=1, - shuffle=True, - ): - self.virtual_epoch_size = virtual_epoch_size - self._current_epoch_start_index = None - self._random_global_indices = None - self.shard_epoch = shard_epoch if shard_epoch is not None else 1 - self.load_next_shard = None - self._epoch_sizes = None - super().__init__( - datasets=datasets, - sampling_ratios=sampling_ratios, - seed=seed, - epoch=epoch, - eval_key=eval_key, - collate_format=collate_format, - virtual_size=virtual_size, - split=split, - shared_collater=shared_collater, - shuffle=shuffle, - ) - - def _setup(self, epoch): - self.virtual_epoch_size = ( - self.virtual_epoch_size - if self.virtual_epoch_size is not None - else self.virtual_size - ) - if self.virtual_epoch_size > self.virtual_size: - logger.warning( - f"virtual epoch size {self.virtual_epoch_size} " - f"is greater than virtual dataset size {self.virtual_size}" - ) - self.virtual_epoch_size = self.virtual_size - self.num_virtual_epochs = math.ceil(self.virtual_size / self.virtual_epoch_size) - self._current_epoch_start_index = self._get_epoch_start_index(epoch) - logger.info( - f"virtual epoch size {self.virtual_epoch_size}; virtual dataset size {self.virtual_size}" - ) - - def _map_epoch_index_to_global(self, index): - index = self._current_epoch_start_index + index - # add randomness - return self._random_global_indices[index] - - @property - def sizes(self): - if self._epoch_sizes is not None: - return self._epoch_sizes - _sizes = super().sizes - indices = self._random_global_indices[ - self._current_epoch_start_index : self._current_epoch_start_index - + len(self) - ] - self._epoch_sizes = _sizes[indices] - # del super()._sizes to save memory - del self._sizes - self._sizes = None - return self._epoch_sizes - - def _get_dataset_and_index(self, index): - i = self._map_epoch_index_to_global(index) - return super()._get_dataset_and_index(i) - - def __len__(self): - return ( - self.virtual_epoch_size - if self._current_epoch_start_index + self.virtual_epoch_size - < self.virtual_size - else self.virtual_size - self._current_epoch_start_index - ) - - def set_epoch(self, epoch): - if self._current_epoch_start_index is None: - # initializing epoch idnices of a virtual dataset - self._setup(epoch) - self._next_virtual_epoch(epoch) - else: - # working on already intialized epoch indices - if epoch == self._cur_epoch: - # re-enter so return - return - self._next_virtual_epoch(epoch) - - def _get_epoch_start_index(self, epoch): - assert epoch >= 1 # fairseq is using 1-based epoch everywhere - return ((epoch - 1) % self.num_virtual_epochs) * self.virtual_epoch_size - - def _next_global_indices(self, epoch): - rng = np.random.RandomState( - [ - int( - hashlib.sha1( - str(self.__class__.__name__).encode("utf-8") - ).hexdigest(), - 16, - ) - % (2**32), - self.seed % (2**32), # global seed - epoch, # epoch index, - ] - ) - del self._random_global_indices - self._random_global_indices = rng.choice( - self.virtual_size, self.virtual_size, replace=False - ) - if self.load_next_shard is None: - self.load_next_shard = False - else: - # increase shard epoch for next loading - self.shard_epoch += 1 - self.load_next_shard = True - logger.info( - "to load next epoch/shard in next load_dataset: " - f"epoch={epoch}/shard_epoch={self.shard_epoch}" - ) - - def _next_virtual_epoch(self, epoch): - index = self._get_epoch_start_index(epoch) - if index == 0 or self._random_global_indices is None: - # need to start from the beginning, - # so call super().set_epoch(epoch) to establish the global virtual indices - logger.info( - "establishing a new set of global virtual indices for " - f"epoch={epoch}/shard_epoch={self.shard_epoch}" - ) - super().set_epoch(epoch) - self._next_global_indices(epoch) - else: - self._cur_epoch = epoch - - # reset cache sizes and ordered_indices for the epoch after moving to a new epoch - self._clean_if_not_none( - [ - self._epoch_sizes, - ] - ) - self._epoch_sizes = None - self._current_epoch_start_index = index diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/multilingual/sampling_method.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/multilingual/sampling_method.py deleted file mode 100644 index 9c826e9c834739e4b7325ec6b7594e922fe49ac7..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/multilingual/sampling_method.py +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import logging -from typing import List - -logger = logging.getLogger(__name__) - - -def uniform(dataset_sizes: List[int]): - return [1.0] * len(dataset_sizes) - - -def temperature_sampling(dataset_sizes, temp): - total_size = sum(dataset_sizes) - return [(size / total_size) ** (1.0 / temp) for size in dataset_sizes] - - -def make_temperature_sampling(temp=1.0): - def sampling_func(dataset_sizes): - return temperature_sampling(dataset_sizes, temp) - - return sampling_func - - -def make_ratio_sampling(ratios): - def sampling_func(dataset_sizes): - return ratios - - return sampling_func - - -class SamplingMethod: - @staticmethod - def add_arguments(parser): - parser.add_argument( - "--sampling-method", - choices=[ - "uniform", - "temperature", - "concat", - "RoundRobin", - ], - type=str, - default="concat", - help="The method to sample data per language pairs", - ) - parser.add_argument( - "--sampling-temperature", - default=1.5, - type=float, - help="only work with --sampling-method temperature", - ) - - @staticmethod - def build_sampler(args, task): - return SamplingMethod(args, task) - - def __init__(self, args, task): - self.args = args - self.task = task - - def is_adaptive(self): - return False - - def sampling_method_selector(self): - args = self.args - logger.info(f"selected sampler: {args.sampling_method}") - if args.sampling_method == "uniform": - return uniform - elif args.sampling_method == "temperature" or self.is_adaptive(): - return make_temperature_sampling(float(args.sampling_temperature)) - else: - # default to concating all data set together - return None diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/nested_dictionary_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/nested_dictionary_dataset.py deleted file mode 100644 index 52e74abddacc923c5e29b0a0c41d7efc85482d3b..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/nested_dictionary_dataset.py +++ /dev/null @@ -1,125 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from collections import OrderedDict - -import torch -from torch.utils.data.dataloader import default_collate - -from . import FairseqDataset - - -def _flatten(dico, prefix=None): - """Flatten a nested dictionary.""" - new_dico = OrderedDict() - if isinstance(dico, dict): - prefix = prefix + "." if prefix is not None else "" - for k, v in dico.items(): - if v is None: - continue - new_dico.update(_flatten(v, prefix + k)) - elif isinstance(dico, list): - for i, v in enumerate(dico): - new_dico.update(_flatten(v, prefix + ".[" + str(i) + "]")) - else: - new_dico = OrderedDict({prefix: dico}) - return new_dico - - -def _unflatten(dico): - """Unflatten a flattened dictionary into a nested dictionary.""" - new_dico = OrderedDict() - for full_k, v in dico.items(): - full_k = full_k.split(".") - node = new_dico - for k in full_k[:-1]: - if k.startswith("[") and k.endswith("]"): - k = int(k[1:-1]) - if k not in node: - node[k] = OrderedDict() - node = node[k] - node[full_k[-1]] = v - return new_dico - - -class NestedDictionaryDataset(FairseqDataset): - def __init__(self, defn, sizes=None): - super().__init__() - self.defn = _flatten(defn) - self.sizes = [sizes] if not isinstance(sizes, (list, tuple)) else sizes - - first = None - for v in self.defn.values(): - if not isinstance( - v, - ( - FairseqDataset, - torch.utils.data.Dataset, - ), - ): - raise ValueError("Expected Dataset but found: {}".format(v.__class__)) - first = first or v - if len(v) > 0: - assert len(v) == len(first), "dataset lengths must match" - - self._len = len(first) - - def __getitem__(self, index): - return OrderedDict((k, ds[index]) for k, ds in self.defn.items()) - - def __len__(self): - return self._len - - def collater(self, samples): - """Merge a list of samples to form a mini-batch. - - Args: - samples (List[dict]): samples to collate - - Returns: - dict: a mini-batch suitable for forwarding with a Model - """ - if len(samples) == 0: - return {} - sample = OrderedDict() - for k, ds in self.defn.items(): - try: - sample[k] = ds.collater([s[k] for s in samples]) - except NotImplementedError: - sample[k] = default_collate([s[k] for s in samples]) - return _unflatten(sample) - - def num_tokens(self, index): - """Return the number of tokens in a sample. This value is used to - enforce ``--max-tokens`` during batching.""" - return max(s[index] for s in self.sizes) - - def size(self, index): - """Return an example's size as a float or tuple. This value is used when - filtering a dataset with ``--max-positions``.""" - if len(self.sizes) == 1: - return self.sizes[0][index] - else: - return (s[index] for s in self.sizes) - - @property - def supports_prefetch(self): - """Whether this dataset supports prefetching.""" - return any(ds.supports_prefetch for ds in self.defn.values()) - - def prefetch(self, indices): - """Prefetch the data required for this epoch.""" - for ds in self.defn.values(): - if getattr(ds, "supports_prefetch", False): - ds.prefetch(indices) - - @property - def can_reuse_epoch_itr_across_epochs(self): - return all(ds.can_reuse_epoch_itr_across_epochs for ds in self.defn.values()) - - def set_epoch(self, epoch): - super().set_epoch(epoch) - for ds in self.defn.values(): - ds.set_epoch(epoch) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/noising.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/noising.py deleted file mode 100644 index e92e83c2cd2e2950d387f93ae8a80acbc12f909f..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/noising.py +++ /dev/null @@ -1,334 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import numpy as np -import torch -from fairseq.data import data_utils - - -class WordNoising(object): - """Generate a noisy version of a sentence, without changing words themselves.""" - - def __init__(self, dictionary, bpe_cont_marker="@@", bpe_end_marker=None): - self.dictionary = dictionary - self.bpe_end = None - if bpe_cont_marker: - self.bpe_end = np.array( - [ - not self.dictionary[i].endswith(bpe_cont_marker) - for i in range(len(self.dictionary)) - ] - ) - elif bpe_end_marker: - self.bpe_end = np.array( - [ - self.dictionary[i].endswith(bpe_end_marker) - for i in range(len(self.dictionary)) - ] - ) - - self.get_word_idx = ( - self._get_bpe_word_idx if self.bpe_end is not None else self._get_token_idx - ) - - def noising(self, x, lengths, noising_prob=0.0): - raise NotImplementedError() - - def _get_bpe_word_idx(self, x): - """ - Given a list of BPE tokens, for every index in the tokens list, - return the index of the word grouping that it belongs to. - For example, for input x corresponding to ["how", "are", "y@@", "ou"], - return [[0], [1], [2], [2]]. - """ - # x: (T x B) - bpe_end = self.bpe_end[x] - - if x.size(0) == 1 and x.size(1) == 1: - # Special case when we only have one word in x. If x = [[N]], - # bpe_end is a scalar (bool) instead of a 2-dim array of bools, - # which makes the sum operation below fail. - return np.array([[0]]) - - # do a reduce front sum to generate word ids - word_idx = bpe_end[::-1].cumsum(0)[::-1] - word_idx = word_idx.max(0)[None, :] - word_idx - return word_idx - - def _get_token_idx(self, x): - """ - This is to extend noising functions to be able to apply to non-bpe - tokens, e.g. word or characters. - """ - x = torch.t(x) - word_idx = np.array([range(len(x_i)) for x_i in x]) - return np.transpose(word_idx) - - -class WordDropout(WordNoising): - """Randomly drop input words. If not passing blank_idx (default is None), - then dropped words will be removed. Otherwise, it will be replaced by the - blank_idx.""" - - def __init__( - self, - dictionary, - default_dropout_prob=0.1, - bpe_cont_marker="@@", - bpe_end_marker=None, - ): - super().__init__(dictionary, bpe_cont_marker, bpe_end_marker) - self.default_dropout_prob = default_dropout_prob - - def noising(self, x, lengths, dropout_prob=None, blank_idx=None): - if dropout_prob is None: - dropout_prob = self.default_dropout_prob - # x: (T x B), lengths: B - if dropout_prob == 0: - return x, lengths - - assert 0 < dropout_prob < 1 - - # be sure to drop entire words - word_idx = self.get_word_idx(x) - sentences = [] - modified_lengths = [] - for i in range(lengths.size(0)): - # Since dropout probabilities need to apply over non-pad tokens, - # it is not trivial to generate the keep mask without consider - # input lengths; otherwise, this could be done outside the loop - - # We want to drop whole words based on word_idx grouping - num_words = max(word_idx[:, i]) + 1 - - # ith example: [x0, x1, ..., eos, pad, ..., pad] - # We should only generate keep probs for non-EOS tokens. Thus if the - # input sentence ends in EOS, the last word idx is not included in - # the dropout mask generation and we append True to always keep EOS. - # Otherwise, just generate the dropout mask for all word idx - # positions. - has_eos = x[lengths[i] - 1, i] == self.dictionary.eos() - if has_eos: # has eos? - keep = np.random.rand(num_words - 1) >= dropout_prob - keep = np.append(keep, [True]) # keep EOS symbol - else: - keep = np.random.rand(num_words) >= dropout_prob - - words = x[: lengths[i], i].tolist() - - # TODO: speed up the following loop - # drop words from the input according to keep - new_s = [ - w if keep[word_idx[j, i]] else blank_idx for j, w in enumerate(words) - ] - new_s = [w for w in new_s if w is not None] - # we need to have at least one word in the sentence (more than the - # start / end sentence symbols) - if len(new_s) <= 1: - # insert at beginning in case the only token left is EOS - # EOS should be at end of list. - new_s.insert(0, words[np.random.randint(0, len(words))]) - assert len(new_s) >= 1 and ( - not has_eos # Either don't have EOS at end or last token is EOS - or (len(new_s) >= 2 and new_s[-1] == self.dictionary.eos()) - ), "New sentence is invalid." - sentences.append(new_s) - modified_lengths.append(len(new_s)) - # re-construct input - modified_lengths = torch.LongTensor(modified_lengths) - modified_x = torch.LongTensor( - modified_lengths.max(), modified_lengths.size(0) - ).fill_(self.dictionary.pad()) - for i in range(modified_lengths.size(0)): - modified_x[: modified_lengths[i], i].copy_(torch.LongTensor(sentences[i])) - - return modified_x, modified_lengths - - -class WordShuffle(WordNoising): - """Shuffle words by no more than k positions.""" - - def __init__( - self, - dictionary, - default_max_shuffle_distance=3, - bpe_cont_marker="@@", - bpe_end_marker=None, - ): - super().__init__(dictionary, bpe_cont_marker, bpe_end_marker) - self.default_max_shuffle_distance = 3 - - def noising(self, x, lengths, max_shuffle_distance=None): - if max_shuffle_distance is None: - max_shuffle_distance = self.default_max_shuffle_distance - # x: (T x B), lengths: B - if max_shuffle_distance == 0: - return x, lengths - - # max_shuffle_distance < 1 will return the same sequence - assert max_shuffle_distance > 1 - - # define noise word scores - noise = np.random.uniform( - 0, - max_shuffle_distance, - size=(x.size(0), x.size(1)), - ) - noise[0] = -1 # do not move start sentence symbol - # be sure to shuffle entire words - word_idx = self.get_word_idx(x) - x2 = x.clone() - for i in range(lengths.size(0)): - length_no_eos = lengths[i] - if x[lengths[i] - 1, i] == self.dictionary.eos(): - length_no_eos = lengths[i] - 1 - # generate a random permutation - scores = word_idx[:length_no_eos, i] + noise[word_idx[:length_no_eos, i], i] - # ensure no reordering inside a word - scores += 1e-6 * np.arange(length_no_eos.item()) - permutation = scores.argsort() - # shuffle words - x2[:length_no_eos, i].copy_( - x2[:length_no_eos, i][torch.from_numpy(permutation)] - ) - return x2, lengths - - -class UnsupervisedMTNoising(WordNoising): - """ - Implements the default configuration for noising in UnsupervisedMT - (github.com/facebookresearch/UnsupervisedMT) - """ - - def __init__( - self, - dictionary, - max_word_shuffle_distance, - word_dropout_prob, - word_blanking_prob, - bpe_cont_marker="@@", - bpe_end_marker=None, - ): - super().__init__(dictionary) - self.max_word_shuffle_distance = max_word_shuffle_distance - self.word_dropout_prob = word_dropout_prob - self.word_blanking_prob = word_blanking_prob - - self.word_dropout = WordDropout( - dictionary=dictionary, - bpe_cont_marker=bpe_cont_marker, - bpe_end_marker=bpe_end_marker, - ) - self.word_shuffle = WordShuffle( - dictionary=dictionary, - bpe_cont_marker=bpe_cont_marker, - bpe_end_marker=bpe_end_marker, - ) - - def noising(self, x, lengths): - # 1. Word Shuffle - noisy_src_tokens, noisy_src_lengths = self.word_shuffle.noising( - x=x, - lengths=lengths, - max_shuffle_distance=self.max_word_shuffle_distance, - ) - # 2. Word Dropout - noisy_src_tokens, noisy_src_lengths = self.word_dropout.noising( - x=noisy_src_tokens, - lengths=noisy_src_lengths, - dropout_prob=self.word_dropout_prob, - ) - # 3. Word Blanking - noisy_src_tokens, noisy_src_lengths = self.word_dropout.noising( - x=noisy_src_tokens, - lengths=noisy_src_lengths, - dropout_prob=self.word_blanking_prob, - blank_idx=self.dictionary.unk(), - ) - - return noisy_src_tokens - - -class NoisingDataset(torch.utils.data.Dataset): - def __init__( - self, - src_dataset, - src_dict, - seed, - noiser=None, - noising_class=UnsupervisedMTNoising, - **kwargs - ): - """ - Wrap a :class:`~torch.utils.data.Dataset` and apply noise to the - samples based on the supplied noising configuration. - - Args: - src_dataset (~torch.utils.data.Dataset): dataset to wrap. - to build self.src_dataset -- - a LanguagePairDataset with src dataset as the source dataset and - None as the target dataset. Should NOT have padding so that - src_lengths are accurately calculated by language_pair_dataset - collate function. - We use language_pair_dataset here to encapsulate the tgt_dataset - so we can re-use the LanguagePairDataset collater to format the - batches in the structure that SequenceGenerator expects. - src_dict (~fairseq.data.Dictionary): source dictionary - seed (int): seed to use when generating random noise - noiser (WordNoising): a pre-initialized :class:`WordNoising` - instance. If this is None, a new instance will be created using - *noising_class* and *kwargs*. - noising_class (class, optional): class to use to initialize a - default :class:`WordNoising` instance. - kwargs (dict, optional): arguments to initialize the default - :class:`WordNoising` instance given by *noiser*. - """ - self.src_dataset = src_dataset - self.src_dict = src_dict - self.seed = seed - self.noiser = ( - noiser - if noiser is not None - else noising_class( - dictionary=src_dict, - **kwargs, - ) - ) - self.sizes = src_dataset.sizes - - def __getitem__(self, index): - """ - Returns a single noisy sample. Multiple samples are fed to the collater - create a noising dataset batch. - """ - src_tokens = self.src_dataset[index] - src_lengths = torch.LongTensor([len(src_tokens)]) - src_tokens = src_tokens.unsqueeze(0) - - # Transpose src tokens to fit expected shape of x in noising function - # (batch size, sequence length) -> (sequence length, batch size) - src_tokens_t = torch.t(src_tokens) - - with data_utils.numpy_seed(self.seed + index): - noisy_src_tokens = self.noiser.noising(src_tokens_t, src_lengths) - - # Transpose back to expected src_tokens format - # (sequence length, 1) -> (1, sequence length) - noisy_src_tokens = torch.t(noisy_src_tokens) - return noisy_src_tokens[0] - - def __len__(self): - """ - The length of the noising dataset is the length of src. - """ - return len(self.src_dataset) - - @property - def supports_prefetch(self): - return self.src_dataset.supports_prefetch - - def prefetch(self, indices): - if self.src_dataset.supports_prefetch: - self.src_dataset.prefetch(indices) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/num_samples_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/num_samples_dataset.py deleted file mode 100644 index 99a17495c701d8a05e0268f98bf453905e11d078..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/num_samples_dataset.py +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from . import FairseqDataset - - -class NumSamplesDataset(FairseqDataset): - def __getitem__(self, index): - return 1 - - def __len__(self): - return 0 - - def collater(self, samples): - return sum(samples) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/numel_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/numel_dataset.py deleted file mode 100644 index ac86dfd2f1d89055de909656d61d6aca85523f00..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/numel_dataset.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import numpy as np -import torch - -from . import BaseWrapperDataset - - -class NumelDataset(BaseWrapperDataset): - def __init__(self, dataset, reduce=False): - super().__init__(dataset) - self.reduce = reduce - - def __getitem__(self, index): - item = self.dataset[index] - if torch.is_tensor(item): - return torch.numel(item) - else: - return np.size(item) - - def __len__(self): - return len(self.dataset) - - def collater(self, samples): - if self.reduce: - return sum(samples) - else: - return torch.tensor(samples) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/offset_tokens_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/offset_tokens_dataset.py deleted file mode 100644 index 6fabbdcdaa1a8f70d8d8c07db4cd53754503c194..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/offset_tokens_dataset.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from . import BaseWrapperDataset - - -class OffsetTokensDataset(BaseWrapperDataset): - def __init__(self, dataset, offset): - super().__init__(dataset) - self.offset = offset - - def __getitem__(self, idx): - return self.dataset[idx] + self.offset diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/pad_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/pad_dataset.py deleted file mode 100644 index b512d370f94d6c4009b1cd42aa3d49279003e59c..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/pad_dataset.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from fairseq.data import data_utils - -from . import BaseWrapperDataset - - -class PadDataset(BaseWrapperDataset): - def __init__(self, dataset, pad_idx, left_pad, pad_length=None): - super().__init__(dataset) - self.pad_idx = pad_idx - self.left_pad = left_pad - self.pad_length = pad_length - - def collater(self, samples): - return data_utils.collate_tokens( - samples, self.pad_idx, left_pad=self.left_pad, pad_to_length=self.pad_length - ) - - -class LeftPadDataset(PadDataset): - def __init__(self, dataset, pad_idx): - super().__init__(dataset, pad_idx, left_pad=True) - - -class RightPadDataset(PadDataset): - def __init__(self, dataset, pad_idx): - super().__init__(dataset, pad_idx, left_pad=False) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/plasma_utils.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/plasma_utils.py deleted file mode 100644 index 459fb8acd789e7b03c70201cb5cb2a9e7dc4f325..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/plasma_utils.py +++ /dev/null @@ -1,197 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - - -import hashlib -import json -import subprocess -import tempfile -from typing import Hashable - -try: - import pyarrow.plasma as plasma - - PYARROW_AVAILABLE = True -except ImportError: - plasma = None - PYARROW_AVAILABLE = False - - -class PlasmaArray: - """ - Wrapper around numpy arrays that automatically moves the data to shared - memory upon serialization. This is particularly helpful when passing numpy - arrays through multiprocessing, so that data is not unnecessarily - duplicated or pickled. - """ - - def __init__(self, array): - super().__init__() - self.array = array - self.disable = array.nbytes < 134217728 # disable for arrays <128MB - self.object_id = None - self.path = None - - # variables with underscores shouldn't be pickled - self._client = None - self._server = None - self._server_tmp = None - self._plasma = None - - @property - def plasma(self): - if self._plasma is None and not self.disable: - self._plasma = plasma - return self._plasma - - def start_server(self): - if self.plasma is None or self._server is not None: - return - assert self.object_id is None - assert self.path is None - self._server_tmp = tempfile.NamedTemporaryFile() - self.path = self._server_tmp.name - self._server = subprocess.Popen( - ["plasma_store", "-m", str(int(1.05 * self.array.nbytes)), "-s", self.path] - ) - - @property - def client(self): - if self._client is None: - assert self.path is not None - self._client = self.plasma.connect(self.path, num_retries=200) - return self._client - - def __getstate__(self): - """Called on pickle load""" - if self.plasma is None: - return self.__dict__ - if self.object_id is None: - self.start_server() - self.object_id = self.client.put(self.array) - state = self.__dict__.copy() - del state["array"] - state["_client"] = None - state["_server"] = None - state["_server_tmp"] = None - state["_plasma"] = None - return state - - def __setstate__(self, state): - """Called on pickle save""" - self.__dict__.update(state) - if self.plasma is None: - return - self.array = self.client.get(self.object_id) - - def __del__(self): - if self._server is not None: - self._server.kill() - self._server = None - self._server_tmp.close() - self._server_tmp = None - - -DEFAULT_PLASMA_PATH = "/tmp/plasma" - - -class PlasmaView: - """Interface to write and read from shared memory. Whereas PlasmaArray writes to plasma on serialization, - PlasmaView writes to shared memory on instantiation.""" - - def __init__(self, array, split_path: str, hash_data: Hashable, plasma_path=None): - """ - Args: - array: numpy array to store. This can be read with ``PlasmaView().array`` - split_path: the path whence the data was read, used for hashing - hash_data: other metadata about the array that can be used to create a unique key. - as of writing, the 3 callers in ``TokenBlockDataset`` use:: - - hash_data = ((block_size, document_sep_len, str(break_mode), len(dataset)), 0|1|2) - - - """ - assert PYARROW_AVAILABLE - assert split_path is not None - if plasma_path is None: - plasma_path = DEFAULT_PLASMA_PATH - - self.path = plasma_path - self.split_path = split_path - self._client = None # Initialize lazily for pickle. plasma clients should not be deep copied or serialized. - self._n = None - - self.object_id = self.get_object_id(self.split_path, hash_data) - try: - self.client.put(array, object_id=self.object_id) - except plasma.PlasmaObjectExists: - pass - - @property - def client(self): - if self._client is None: - self._client = plasma.connect(self.path, num_retries=200) - return self._client - - @property - def array(self): - """Fetch a read only view of an np.array, stored in plasma.""" - ret = self.client.get(self.object_id) - return ret - - @staticmethod - def get_object_id(split_path: str, hash_data: Hashable): - """Returns plasma.ObjectID from hashing split_path and object_num.""" - hash = hashlib.blake2b(bytes(split_path, "utf-8"), digest_size=20) - harg = json.dumps(hash_data).encode("utf-8") - hash.update(harg) - return plasma.ObjectID(hash.digest()) - - def __getstate__(self): - """Called on pickle save""" - self.disconnect() - state = self.__dict__.copy() - assert state["_client"] is None - assert "object_id" in state - return state - - def __setstate__(self, state): - """Called on pickle load""" - self.__dict__.update(state) - - def __del__(self): - self.disconnect() - - def disconnect(self): - if self._client is not None: - self._client.disconnect() - self._client = None - - def __len__(self): - """Save reads by caching len""" - if self._n is None: - self._n = len(self.array) - return self._n - - -GB100 = (1024**3) * 100 - - -class PlasmaStore: - def __init__(self, path=DEFAULT_PLASMA_PATH, nbytes: int = GB100): - - self.server = self.start(path, nbytes) - - def __del__(self): - self.server.kill() - - @staticmethod - def start(path=DEFAULT_PLASMA_PATH, nbytes: int = GB100) -> subprocess.Popen: - if not PYARROW_AVAILABLE: - raise ImportError("please run pip install pyarrow to use --use_plasma_view") - # best practice is to allocate more space than we need. The limitation seems to be the size of /dev/shm - _server = subprocess.Popen(["plasma_store", "-m", str(nbytes), "-s", path]) - plasma.connect(path, num_retries=200) # If we can't connect we fail immediately - return _server diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/prepend_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/prepend_dataset.py deleted file mode 100644 index ad74784d2d7920e4a6225282d95543ce16ea50d9..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/prepend_dataset.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import numpy as np -import torch - -from . import BaseWrapperDataset - - -class PrependDataset(BaseWrapperDataset): - def __init__(self, dataset, prepend_getter, ensure_first_token_is=None): - super().__init__(dataset) - self.prepend_getter = prepend_getter - self.ensure_first_token = ensure_first_token_is - - def __getitem__(self, idx): - item = self.dataset[idx] - is_tuple = isinstance(item, tuple) - src = item[0] if is_tuple else item - - assert self.ensure_first_token is None or src[0] == self.ensure_first_token - prepend_idx = self.prepend_getter(self.dataset, idx) - assert isinstance(prepend_idx, int) - src[0] = prepend_idx - item = tuple((src,) + item[1:]) if is_tuple else src - return item diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/prepend_token_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/prepend_token_dataset.py deleted file mode 100644 index fd1331f4c44c1595eb9bb78baa0cf5cf3bcce9ad..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/prepend_token_dataset.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import numpy as np -import torch - -from . import BaseWrapperDataset - - -class PrependTokenDataset(BaseWrapperDataset): - def __init__(self, dataset, token=None): - super().__init__(dataset) - self.token = token - if token is not None: - self._sizes = np.array(dataset.sizes) + 1 - else: - self._sizes = dataset.sizes - - def __getitem__(self, idx): - item = self.dataset[idx] - if self.token is not None: - item = torch.cat([item.new([self.token]), item]) - return item - - @property - def sizes(self): - return self._sizes - - def num_tokens(self, index): - n = self.dataset.num_tokens(index) - if self.token is not None: - n += 1 - return n - - def size(self, index): - n = self.dataset.size(index) - if self.token is not None: - n += 1 - return n diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/raw_label_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/raw_label_dataset.py deleted file mode 100644 index d054904f419bd64855d33a2a770b43f671c7c8d8..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/raw_label_dataset.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import torch - -from . import FairseqDataset - - -class RawLabelDataset(FairseqDataset): - def __init__(self, labels): - super().__init__() - self.labels = labels - - def __getitem__(self, index): - return self.labels[index] - - def __len__(self): - return len(self.labels) - - def collater(self, samples): - return torch.tensor(samples) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/replace_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/replace_dataset.py deleted file mode 100644 index 5aac2ba96bee0a8bb65f4c9e56fa0b17248ee1d9..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/replace_dataset.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from . import BaseWrapperDataset - - -class ReplaceDataset(BaseWrapperDataset): - """Replaces tokens found in the dataset by a specified replacement token - - Args: - dataset (~torch.utils.data.Dataset): dataset to replace tokens in - replace_map(Dictionary[int,int]): map of token to replace -> replacement token - offsets (List[int]): do not replace tokens before (from left if pos, right if neg) this offset. should be - as many as the number of objects returned by the underlying dataset __getitem__ method. - """ - - def __init__(self, dataset, replace_map, offsets): - super().__init__(dataset) - assert len(replace_map) > 0 - self.replace_map = replace_map - self.offsets = offsets - - def __getitem__(self, index): - item = self.dataset[index] - is_tuple = isinstance(item, tuple) - srcs = item if is_tuple else [item] - - for offset, src in zip(self.offsets, srcs): - for k, v in self.replace_map.items(): - src_off = src[offset:] if offset >= 0 else src[:offset] - src_off.masked_fill_(src_off == k, v) - - item = srcs if is_tuple else srcs[0] - return item diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/resampling_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/resampling_dataset.py deleted file mode 100644 index adb960d6308d3b027a7752a8989c7d0af14550a6..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/resampling_dataset.py +++ /dev/null @@ -1,138 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import logging - -import numpy as np -from fairseq.data import BaseWrapperDataset, plasma_utils - -logger = logging.getLogger(__name__) - - -class ResamplingDataset(BaseWrapperDataset): - """Randomly samples from a given dataset at each epoch. - - Sampling is done with or without replacement, depending on the "replace" - parameter. - - Optionally, the epoch size can be rescaled. This is potentially desirable - to increase per-epoch coverage of the base dataset (since sampling with - replacement means that many items in the dataset will be left out). In the - case of sampling without replacement, size_ratio should be strictly less - than 1. - - Args: - dataset (~torch.utils.data.Dataset): dataset on which to sample. - weights (List[float]): list of probability weights - (default: None, which corresponds to uniform sampling). - replace (bool): sampling mode; True for "with replacement", or False - for "without replacement" (default: True) - size_ratio (float): the ratio to subsample to; must be positive - (default: 1.0). - batch_by_size (bool): whether or not to batch by sequence length - (default: True). - seed (int): RNG seed to use (default: 0). - epoch (int): starting epoch number (default: 1). - """ - - def __init__( - self, - dataset, - weights=None, - replace=True, - size_ratio=1.0, - batch_by_size=True, - seed=0, - epoch=1, - ): - super().__init__(dataset) - - if weights is None: - self.weights = None - - else: - assert len(weights) == len(dataset) - weights_arr = np.array(weights, dtype=np.float64) - weights_arr /= weights_arr.sum() - self.weights = plasma_utils.PlasmaArray(weights_arr) - - self.replace = replace - - assert size_ratio > 0.0 - if not self.replace: - assert size_ratio < 1.0 - self.size_ratio = float(size_ratio) - self.actual_size = np.ceil(len(dataset) * self.size_ratio).astype(int) - - self.batch_by_size = batch_by_size - self.seed = seed - - self._cur_epoch = None - self._cur_indices = None - - self.set_epoch(epoch) - - def __getitem__(self, index): - return self.dataset[self._cur_indices.array[index]] - - def __len__(self): - return self.actual_size - - @property - def sizes(self): - if isinstance(self.dataset.sizes, list): - return [s[self._cur_indices.array] for s in self.dataset.sizes] - return self.dataset.sizes[self._cur_indices.array] - - def num_tokens(self, index): - return self.dataset.num_tokens(self._cur_indices.array[index]) - - def size(self, index): - return self.dataset.size(self._cur_indices.array[index]) - - def ordered_indices(self): - if self.batch_by_size: - order = [ - np.arange(len(self)), - self.sizes, - ] # No need to handle `self.shuffle == True` - return np.lexsort(order) - else: - return np.arange(len(self)) - - def prefetch(self, indices): - self.dataset.prefetch(self._cur_indices.array[indices]) - - @property - def can_reuse_epoch_itr_across_epochs(self): - return False - - def set_epoch(self, epoch): - logger.debug("ResamplingDataset.set_epoch: {}".format(epoch)) - super().set_epoch(epoch) - - if epoch == self._cur_epoch: - return - - self._cur_epoch = epoch - - # Generate a weighted sample of indices as a function of the - # random seed and the current epoch. - - rng = np.random.RandomState( - [ - 42, # magic number - self.seed % (2**32), # global seed - self._cur_epoch, # epoch index - ] - ) - self._cur_indices = plasma_utils.PlasmaArray( - rng.choice( - len(self.dataset), - self.actual_size, - replace=self.replace, - p=(None if self.weights is None else self.weights.array), - ) - ) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/roll_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/roll_dataset.py deleted file mode 100644 index a2915eeb3e8fb4dfb4b2bb33e0464ad0783d854c..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/roll_dataset.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import torch - -from . import BaseWrapperDataset - - -class RollDataset(BaseWrapperDataset): - def __init__(self, dataset, shifts): - super().__init__(dataset) - self.shifts = shifts - - def __getitem__(self, index): - item = self.dataset[index] - return torch.roll(item, self.shifts) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/round_robin_zip_datasets.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/round_robin_zip_datasets.py deleted file mode 100644 index 2cb7447ea955a7c3ae7372f09ee426c08acd430e..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/round_robin_zip_datasets.py +++ /dev/null @@ -1,160 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import logging -from collections import OrderedDict -from typing import Dict, Sequence - -import numpy as np - -from . import FairseqDataset, LanguagePairDataset - -logger = logging.getLogger(__name__) - - -class RoundRobinZipDatasets(FairseqDataset): - """Zip multiple :class:`~fairseq.data.FairseqDataset` instances together. - - Shorter datasets are repeated in a round-robin fashion to match the length - of the longest one. - - Args: - datasets (Dict[~fairseq.data.FairseqDataset]): a dictionary of - :class:`~fairseq.data.FairseqDataset` instances. - eval_key (str, optional): a key used at evaluation time that causes - this instance to pass-through batches from *datasets[eval_key]*. - """ - - def __init__(self, datasets, eval_key=None): - super().__init__() - if isinstance(datasets, dict): - datasets = OrderedDict(datasets) - assert isinstance(datasets, OrderedDict) - assert datasets, "Can't make a RoundRobinZipDatasets out of nothing" - for dataset in datasets.values(): - assert isinstance(dataset, FairseqDataset) - - self.datasets = datasets - self.eval_key = eval_key - - self.longest_dataset_key = max(datasets, key=lambda k: len(datasets[k])) - self.longest_dataset = datasets[self.longest_dataset_key] - self._ordered_indices: Dict[str, Sequence[int]] = None - - def _map_index(self, key, index): - assert ( - self._ordered_indices is not None - ), "Must call RoundRobinZipDatasets.ordered_indices() first" - o = self._ordered_indices[key] - return o[index % len(o)] - - def __getitem__(self, index): - if self.eval_key is None: - return OrderedDict( - [ - (key, dataset[self._map_index(key, index)]) - for key, dataset in self.datasets.items() - ] - ) - else: - # at evaluation time it's useful to pass-through batches from a single key - return self.datasets[self.eval_key][self._map_index(self.eval_key, index)] - - def __len__(self): - if self._ordered_indices is not None: - return len(self._ordered_indices[self.longest_dataset_key]) - return len(self.longest_dataset) - - def collater(self, samples): - """Merge a list of samples to form a mini-batch.""" - if len(samples) == 0: - return None - if self.eval_key is None: - return OrderedDict( - [ - (key, dataset.collater([sample[key] for sample in samples])) - for key, dataset in self.datasets.items() - ] - ) - else: - # at evaluation time it's useful to pass-through batches from a single key - return self.datasets[self.eval_key].collater(samples) - - def num_tokens(self, index): - """Return an example's length (number of tokens), used for batching.""" - # TODO make it configurable whether to use max() or sum() here - return max( - dataset.num_tokens(self._map_index(key, index)) - for key, dataset in self.datasets.items() - ) - - def size(self, index): - """Return an example's size as a float or tuple. This value is used when - filtering a dataset with ``--max-positions``.""" - return { - key: dataset.size(self._map_index(key, index)) - for key, dataset in self.datasets.items() - } - - def ordered_indices(self): - """Ordered indices for batching.""" - if self._ordered_indices is None: - # Call the underlying dataset's ordered_indices() here, so that we - # get the same random ordering as we would have from using the - # underlying sub-datasets directly. - self._ordered_indices = OrderedDict( - [ - (key, dataset.ordered_indices()) - for key, dataset in self.datasets.items() - ] - ) - return np.arange(len(self)) - - def filter_indices_by_size(self, indices, max_positions=None): - """ - Filter each sub-dataset independently, then update the round robin to work - on the filtered sub-datasets. - """ - - def _deep_until_language_pair(dataset): - if isinstance(dataset, LanguagePairDataset): - return dataset - if hasattr(dataset, "tgt_dataset"): - return _deep_until_language_pair(dataset.tgt_dataset) - if hasattr(dataset, "dataset"): - return _deep_until_language_pair(dataset.dataset) - raise Exception(f"Don't know how to unwrap this dataset: {dataset}") - - if not isinstance(max_positions, dict): - max_positions = {k: max_positions for k in self.datasets.keys()} - ignored_some = False - for key, dataset in self.datasets.items(): - dataset = _deep_until_language_pair(dataset) - self._ordered_indices[key], ignored = dataset.filter_indices_by_size( - self._ordered_indices[key], max_positions[key] - ) - if len(ignored) > 0: - ignored_some = True - logger.warning( - f"{len(ignored)} samples from {key} have invalid sizes and will be skipped, " - f"max_positions={max_positions[key]}, first few sample ids={ignored[:10]}" - ) - # Since we are modifying in place the _ordered_indices, - # it's not possible anymore to return valid ignored indices. - # Hopefully the extra debug information print above should be enough to debug. - # Ideally we would receive ignore_invalid_inputs so that we could have - # a proper error message. - return (np.arange(len(self)), [0] if ignored_some else []) - - @property - def supports_prefetch(self): - return all( - getattr(dataset, "supports_prefetch", False) - for dataset in self.datasets.values() - ) - - def prefetch(self, indices): - for key, dataset in self.datasets.items(): - dataset.prefetch([self._map_index(key, index) for index in indices]) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/shorten_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/shorten_dataset.py deleted file mode 100644 index 6ebb5d88feb3f29d1512a0873df304915d051209..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/shorten_dataset.py +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import numpy as np -from fairseq.data import data_utils - -from . import BaseWrapperDataset - - -class TruncateDataset(BaseWrapperDataset): - """Truncate a sequence by returning the first truncation_length tokens""" - - def __init__(self, dataset, truncation_length): - super().__init__(dataset) - assert truncation_length is not None - self.truncation_length = truncation_length - self.dataset = dataset - - def __getitem__(self, index): - item = self.dataset[index] - item_len = item.size(0) - if item_len > self.truncation_length: - item = item[: self.truncation_length] - return item - - @property - def sizes(self): - return np.minimum(self.dataset.sizes, self.truncation_length) - - def __len__(self): - return len(self.dataset) - - -class RandomCropDataset(TruncateDataset): - """Truncate a sequence by returning a random crop of truncation_length tokens""" - - def __init__(self, dataset, truncation_length, seed=1): - super().__init__(dataset, truncation_length) - self.seed = seed - self.epoch = 0 - - @property - def can_reuse_epoch_itr_across_epochs(self): - return True # only the crop changes, not item sizes - - def set_epoch(self, epoch, **unused): - super().set_epoch(epoch) - self.epoch = epoch - - def __getitem__(self, index): - with data_utils.numpy_seed(self.seed, self.epoch, index): - item = self.dataset[index] - item_len = item.size(0) - excess = item_len - self.truncation_length - if excess > 0: - start_idx = np.random.randint(0, excess) - item = item[start_idx : start_idx + self.truncation_length] - return item - - -def maybe_shorten_dataset( - dataset, - split, - shorten_data_split_list, - shorten_method, - tokens_per_sample, - seed, -): - truncate_split = ( - split in shorten_data_split_list.split(",") or len(shorten_data_split_list) == 0 - ) - if shorten_method == "truncate" and truncate_split: - dataset = TruncateDataset(dataset, tokens_per_sample) - elif shorten_method == "random_crop" and truncate_split: - dataset = RandomCropDataset(dataset, tokens_per_sample, seed) - return dataset diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/sort_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/sort_dataset.py deleted file mode 100644 index b3890e7279e1f26db2e48ec0a91c639e9299d60f..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/sort_dataset.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import numpy as np - -from . import BaseWrapperDataset - - -class SortDataset(BaseWrapperDataset): - def __init__(self, dataset, sort_order): - super().__init__(dataset) - if not isinstance(sort_order, (list, tuple)): - sort_order = [sort_order] - self.sort_order = sort_order - - assert all(len(so) == len(dataset) for so in sort_order) - - def ordered_indices(self): - return np.lexsort(self.sort_order) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/strip_token_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/strip_token_dataset.py deleted file mode 100644 index cae39ba4d2f8106398eccd7eb0cf5c2194ec0db5..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/strip_token_dataset.py +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from . import BaseWrapperDataset - - -class StripTokenDataset(BaseWrapperDataset): - def __init__(self, dataset, id_to_strip): - super().__init__(dataset) - self.id_to_strip = id_to_strip - - def __getitem__(self, index): - item = self.dataset[index] - while len(item) > 0 and item[-1] == self.id_to_strip: - item = item[:-1] - while len(item) > 0 and item[0] == self.id_to_strip: - item = item[1:] - return item diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/subsample_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/subsample_dataset.py deleted file mode 100644 index 9d01b1ab4eefe0fd5ae51f353a58f2dc8ca1ddb7..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/subsample_dataset.py +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import logging - -import numpy as np - -from . import BaseWrapperDataset - -logger = logging.getLogger(__name__) - - -class SubsampleDataset(BaseWrapperDataset): - """Subsamples a given dataset by a specified ratio. Subsampling is done on the number of examples - - Args: - dataset (~torch.utils.data.Dataset): dataset to subsample - size_ratio(float): the ratio to subsample to. must be between 0 and 1 (exclusive) - """ - - def __init__(self, dataset, size_ratio, shuffle=False): - super().__init__(dataset) - assert size_ratio < 1 - self.actual_size = np.ceil(len(dataset) * size_ratio).astype(int) - self.indices = np.random.choice( - list(range(len(self.dataset))), self.actual_size, replace=False - ) - self.shuffle = shuffle - logger.info( - "subsampled dataset from {} to {} (ratio={})".format( - len(self.dataset), self.actual_size, size_ratio - ) - ) - - def __getitem__(self, index): - return self.dataset[self.indices[index]] - - def __len__(self): - return self.actual_size - - def collater(self, samples): - return self.dataset.collater(samples) - - @property - def sizes(self): - return self.dataset.sizes[self.indices] - - @property - def name(self): - return self.dataset.name - - def num_tokens(self, index): - return self.dataset.num_tokens(self.indices[index]) - - def size(self, index): - return self.dataset.size(self.indices[index]) - - def ordered_indices(self): - """Return an ordered list of indices. Batches will be constructed based - on this order.""" - if self.shuffle: - order = [np.random.permutation(len(self))] - else: - order = [np.arange(len(self))] - order.append(self.sizes) - return np.lexsort(order) - - def prefetch(self, indices): - self.dataset.prefetch(self.indices[indices]) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/text_compressor.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/text_compressor.py deleted file mode 100644 index d699f2ea296f33cdc37ca152ab225d09cb04b5ea..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/text_compressor.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from enum import Enum - - -class TextCompressionLevel(Enum): - none = 0 - low = 1 - high = 2 - - -class TextCompressor(object): - def __init__( - self, level: TextCompressionLevel, max_input_byte_length: int = 2**16 - ): - self.level = level - self.max_input_length = max_input_byte_length - - def compress(self, text: str) -> bytes: - if self.level == TextCompressionLevel.low: - import zlib - - # zlib: built-in, fast - return zlib.compress(text.encode(), level=0) - elif self.level == TextCompressionLevel.high: - try: - import unishox2 - - # unishox2: optimized for short text but slower - except ImportError: - raise ImportError( - "Please install unishox2 for the text compression feature: " - "pip install unishox2-py3" - ) - assert len(text.encode()) <= self.max_input_length - return unishox2.compress(text)[0] - else: - return text.encode() - - def decompress(self, compressed: bytes) -> str: - if self.level == TextCompressionLevel.low: - import zlib - - return zlib.decompress(compressed).decode() - elif self.level == TextCompressionLevel.high: - try: - import unishox2 - except ImportError: - raise ImportError( - "Please install unishox2 for the text compression feature: " - "pip install unishox2-py3" - ) - return unishox2.decompress(compressed, self.max_input_length) - else: - return compressed.decode() diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/token_block_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/token_block_dataset.py deleted file mode 100644 index bebf7a49bb61cf43025e00a3535211bbd93d2d72..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/token_block_dataset.py +++ /dev/null @@ -1,207 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from typing import Tuple - -import numpy as np -import torch -from fairseq.data import FairseqDataset, plasma_utils -from fairseq.data.indexed_dataset import best_fitting_int_dtype - - -class TokenBlockDataset(FairseqDataset): - """Break a Dataset of tokens into blocks. - - Args: - dataset (~torch.utils.data.Dataset): dataset to break into blocks - sizes (List[int]): sentence lengths (required for 'complete' and 'eos') - block_size (int): maximum block size (ignored in 'eos' break mode) - break_mode (str, optional): Mode used for breaking tokens. Values can - be one of: - - 'none': break tokens into equally sized blocks (up to block_size) - - 'complete': break tokens into blocks (up to block_size) such that - blocks contains complete sentences, although block_size may be - exceeded if some sentences exceed block_size - - 'complete_doc': similar to 'complete' mode, but do not - cross document boundaries - - 'eos': each block contains one sentence (block_size is ignored) - include_targets (bool, optional): return next tokens as targets - (default: False). - document_sep_len (int, optional): document separator size (required for - 'complete_doc' break mode). Typically 1 if the sentences have eos - and 0 otherwise. - """ - - def __init__( - self, - dataset, - sizes, - block_size, - pad, - eos, - break_mode=None, - include_targets=False, - document_sep_len=1, - use_plasma_view=False, - split_path=None, - plasma_path=None, - ): - - super().__init__() - self.dataset = dataset - self.pad = pad - self.eos = eos - self.include_targets = include_targets - - assert len(dataset) > 0 - - assert len(dataset) == len(sizes) - _sizes, block_to_dataset_index, slice_indices = self._build_slice_indices( - sizes, break_mode, document_sep_len, block_size - ) - if use_plasma_view: - plasma_id = (block_size, document_sep_len, str(break_mode), len(dataset)) - self._slice_indices = plasma_utils.PlasmaView( - slice_indices, split_path, (plasma_id, 0), plasma_path=plasma_path - ) - self._sizes = plasma_utils.PlasmaView( - _sizes, split_path, (plasma_id, 1), plasma_path=plasma_path - ) - self._block_to_dataset_index = plasma_utils.PlasmaView( - block_to_dataset_index, - split_path, - (plasma_id, 2), - plasma_path=plasma_path, - ) - else: - self._slice_indices = plasma_utils.PlasmaArray(slice_indices) - self._sizes = plasma_utils.PlasmaArray(_sizes) - self._block_to_dataset_index = plasma_utils.PlasmaArray( - block_to_dataset_index - ) - - @staticmethod - def _build_slice_indices( - sizes, break_mode, document_sep_len, block_size - ) -> Tuple[np.ndarray]: - """Use token_block_utils_fast to build arrays for indexing into self.dataset""" - try: - from fairseq.data.token_block_utils_fast import ( - _get_block_to_dataset_index_fast, - _get_slice_indices_fast, - ) - except ImportError: - raise ImportError( - "Please build Cython components with: `pip install --editable .` " - "or `python setup.py build_ext --inplace`" - ) - - if isinstance(sizes, list): - sizes = np.array(sizes, dtype=np.int64) - else: - if torch.is_tensor(sizes): - sizes = sizes.numpy() - sizes = sizes.astype(np.int64) - - break_mode = break_mode if break_mode is not None else "none" - - # For "eos" break-mode, block_size is not required parameters. - if break_mode == "eos" and block_size is None: - block_size = 0 - - slice_indices = _get_slice_indices_fast( - sizes, str(break_mode), block_size, document_sep_len - ) - _sizes = slice_indices[:, 1] - slice_indices[:, 0] - - # build index mapping block indices to the underlying dataset indices - if break_mode == "eos": - # much faster version for eos break mode - block_to_dataset_index = np.stack( - [ - np.arange(len(sizes)), # starting index in dataset - np.zeros( - len(sizes), dtype=np.compat.long - ), # starting offset within starting index - np.arange(len(sizes)), # ending index in dataset - ], - 1, - ) - else: - block_to_dataset_index = _get_block_to_dataset_index_fast( - sizes, - slice_indices, - ) - size_dtype = np.uint16 if block_size < 65535 else np.uint32 - num_tokens = slice_indices[-1].max() - slice_indices_dtype = best_fitting_int_dtype(num_tokens) - slice_indices = slice_indices.astype(slice_indices_dtype) - _sizes = _sizes.astype(size_dtype) - block_to_dataset_index = block_to_dataset_index.astype(slice_indices_dtype) - return _sizes, block_to_dataset_index, slice_indices - - @property - def slice_indices(self): - return self._slice_indices.array - - @property - def sizes(self): - return self._sizes.array - - @property - def block_to_dataset_index(self): - return self._block_to_dataset_index.array - - def attr(self, attr: str, index: int): - start_ds_idx, _, _ = self.block_to_dataset_index[index] - return self.dataset.attr(attr, start_ds_idx) - - def __getitem__(self, index): - start_ds_idx, start_offset, end_ds_idx = self.block_to_dataset_index[index] - - buffer = torch.cat( - [self.dataset[idx] for idx in range(start_ds_idx, end_ds_idx + 1)] - ) - slice_s, slice_e = self.slice_indices[index] - length = slice_e - slice_s - s, e = start_offset, start_offset + length - item = buffer[s:e] - - if self.include_targets: - # *target* is the original sentence (=item) - # *source* is shifted right by 1 (maybe left-padded with eos) - # *past_target* is shifted right by 2 (left-padded as needed) - if s == 0: - source = torch.cat([item.new([self.eos]), buffer[0 : e - 1]]) - past_target = torch.cat( - [item.new([self.pad, self.eos]), buffer[0 : e - 2]] - ) - else: - source = buffer[s - 1 : e - 1] - if s == 1: - past_target = torch.cat([item.new([self.eos]), buffer[0 : e - 2]]) - else: - past_target = buffer[s - 2 : e - 2] - - return source, item, past_target - - return item - - def __len__(self): - return len(self.slice_indices) - - @property - def supports_prefetch(self): - return getattr(self.dataset, "supports_prefetch", False) - - def prefetch(self, indices): - self.dataset.prefetch( - { - ds_idx - for index in indices - for start_ds_idx, _, end_ds_idx in [self.block_to_dataset_index[index]] - for ds_idx in range(start_ds_idx, end_ds_idx + 1) - } - ) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/token_block_utils_fast.pyx b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/token_block_utils_fast.pyx deleted file mode 100644 index 61fb0112442a1e3ea74e3eab420b1f8697ded60f..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/token_block_utils_fast.pyx +++ /dev/null @@ -1,187 +0,0 @@ -# cython: language_level=3 -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from itertools import chain - -import numpy as np -import torch - -cimport cython -cimport numpy as np -from libc.math cimport ceil -from libc.stdint cimport int32_t, int64_t - -DTYPE = np.int64 -ctypedef int64_t DTYPE_t - - -@cython.boundscheck(False) -@cython.wraparound(False) -@cython.nonecheck(False) -cdef np.ndarray[DTYPE_t, ndim=2] _get_slice_indices_none_mode(np.ndarray[DTYPE_t, ndim=1] sizes, int block_size): - cdef DTYPE_t total_size = sizes.sum() - cdef DTYPE_t length = <DTYPE_t> ceil(total_size / <double> block_size) - cdef np.ndarray[DTYPE_t, ndim=2] slice_indices = np.zeros([length, 2], dtype=DTYPE) - cdef DTYPE_t[:, :] slice_indices_view = slice_indices - cdef DTYPE_t i - cdef DTYPE_t start - cdef DTYPE_t end - for i in range(length): - start = i * block_size - end = min(start + block_size, total_size) - slice_indices_view[i][0] = start - slice_indices_view[i][1] = end - return slice_indices - - -cdef np.ndarray[DTYPE_t, ndim=2] _fast_convert_to_np_array(list list_of_list): - """ - Faster function to convert DTYPE_t list of list. - Only fast when there are huge number of rows and low number of columns. - """ - cdef np.ndarray[DTYPE_t, ndim=1] flat = np.fromiter(chain.from_iterable(list_of_list), DTYPE, -1) - return flat.reshape((len(list_of_list), -1)) - - -@cython.boundscheck(False) -@cython.wraparound(False) -@cython.nonecheck(False) -cpdef np.ndarray[DTYPE_t, ndim=2] _get_slice_indices_fast(np.ndarray[DTYPE_t, ndim=1] sizes, str break_mode, int block_size, int document_sep_len): - cdef DTYPE_t tok_idx = 0 - cdef DTYPE_t sz_idx = 0 - cdef DTYPE_t curr_size = 0 - cdef DTYPE_t i = 0 - cdef DTYPE_t length - cdef DTYPE_t total_size - cdef DTYPE_t[:] sizes_view = sizes - cdef np.ndarray[DTYPE_t, ndim=2] slice_indices - cdef list slice_indices_list = [] - - if break_mode is None or break_mode == 'none': - slice_indices = _get_slice_indices_none_mode(sizes, block_size) - elif break_mode == 'complete': - while sz_idx < len(sizes_view): - if curr_size + sizes_view[sz_idx] <= block_size or curr_size == 0: - curr_size += sizes_view[sz_idx] - sz_idx += 1 - else: - slice_indices_list.append((tok_idx, tok_idx + curr_size)) - tok_idx += curr_size - curr_size = 0 - if curr_size > 0: - slice_indices_list.append((tok_idx, tok_idx + curr_size)) - slice_indices = _fast_convert_to_np_array(slice_indices_list) - elif break_mode == 'complete_doc': - while sz_idx < len(sizes_view): - if ( - (curr_size + sizes_view[sz_idx] <= block_size or curr_size == 0) - # an empty sentence indicates end-of-document: - and sizes_view[sz_idx] != document_sep_len - ): - curr_size += sizes_view[sz_idx] - sz_idx += 1 - else: - # Only keep non-empty documents. - if curr_size > 1: - slice_indices_list.append((tok_idx, tok_idx + curr_size)) - tok_idx += curr_size - curr_size = 0 - if sizes_view[sz_idx] == document_sep_len: - tok_idx += sizes_view[sz_idx] - sz_idx += 1 - if curr_size > 1: - slice_indices_list.append((tok_idx, tok_idx + curr_size)) - slice_indices = _fast_convert_to_np_array(slice_indices_list) - elif break_mode == 'eos': - slice_indices = np.zeros((len(sizes), 2), dtype=DTYPE) - cumsum = sizes.cumsum(axis=0) - slice_indices[1:, 0] = cumsum[:cumsum.shape[0] - 1] - slice_indices[:, 1] = cumsum - else: - raise ValueError('Invalid break_mode: ' + break_mode) - return slice_indices - - -@cython.boundscheck(False) -@cython.wraparound(False) -@cython.nonecheck(False) -cpdef np.ndarray[DTYPE_t, ndim=2] _get_block_to_dataset_index_fast(np.ndarray[DTYPE_t, ndim=1] sizes, np.ndarray[DTYPE_t, ndim=2] slice_indices): - cdef DTYPE_t start_ds_idx - cdef DTYPE_t start_offset - cdef DTYPE_t end_ds_idx - cdef DTYPE_t i - cdef DTYPE_t s - cdef DTYPE_t e - cdef DatasetSearcher ds = DatasetSearcher(sizes) - cdef np.ndarray[DTYPE_t, ndim=2] block_to_dataset_index = np.zeros([len(slice_indices), 3], dtype=DTYPE) - cdef DTYPE_t[:, :] block_to_dataset_index_view = block_to_dataset_index - cdef DTYPE_t[:, :] slice_indices_view = slice_indices - cdef Py_ssize_t x_max = slice_indices.shape[0] - - for i in range(x_max): - s = slice_indices_view[i][0] - e = slice_indices_view[i][1] - ds.seek(s) - start_ds_idx = ds.current_index - start_offset = ds.current_offset - if e <= s: - end_ds_idx = start_ds_idx - else: - ds.seek(e - 1) - end_ds_idx = ds.current_index - block_to_dataset_index_view[i][0] = start_ds_idx # starting index in dataset - block_to_dataset_index_view[i][1] = start_offset # starting offset within starting index - block_to_dataset_index_view[i][2] = end_ds_idx # ending index in dataset - return block_to_dataset_index - - -cdef class DatasetSearcher(object): - """Helper for mapping "flat" indices to indices and offsets in an - underlying dataset.""" - cdef DTYPE_t current_i - cdef DTYPE_t current_offset - cdef DTYPE_t current_index - cdef DTYPE_t[:] sizes - - def __init__(self, DTYPE_t[:] sizes): - self.sizes = sizes - self.reset() - - cdef reset(self): - self.current_offset = 0 # offset within current index in underlying dataset - self.current_i = 0 # "flat" index - self.current_index = 0 # index in underlying dataset - - @cython.boundscheck(False) - @cython.wraparound(False) - @cython.nonecheck(False) - cdef int step(self, DTYPE_t i): - cdef DTYPE_t to_consume - cdef DTYPE_t remaining - if i < self.current_i: - self.reset() - if i > self.current_i: - to_consume = i - self.current_i - remaining = self.sizes[self.current_index] - self.current_offset - if remaining > to_consume: - self.current_offset += to_consume - self.current_i += to_consume - else: - assert remaining >= 0 - self.current_i += remaining - self.current_index += 1 - self.current_offset = 0 - return 1 - return 0 - - @cython.boundscheck(False) - @cython.wraparound(False) - @cython.nonecheck(False) - cdef seek(self, DTYPE_t i): - cdef int not_done = 1 - while not_done == 1: - not_done = self.step(i) - assert self.current_i == i diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/transform_eos_concat_langpair_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/transform_eos_concat_langpair_dataset.py deleted file mode 100644 index 2a5050e9a0d6e24285827450a3c49549faf0f72f..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/transform_eos_concat_langpair_dataset.py +++ /dev/null @@ -1,138 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import logging - -import torch -from fairseq.data import ConcatDataset -from torch.utils.data.dataloader import default_collate - -logger = logging.getLogger(__name__) - - -class TransformEosConcatLangPairDataset(ConcatDataset): - """ - It is a combination of TransformEosLangPairDataset and ConcatDataset for multiple LangPairDataset datasets. - Assume all datasets share the same src_eos, tgt_bos, left_pad_source and left_pad_target - """ - - def __init__( - self, - datasets, - src_eos, - tgt_bos, - new_src_eos=None, - new_tgt_bos=None, - ): - super().__init__(datasets) - if new_src_eos is not None: - assert len(new_src_eos) == len(datasets) - else: - new_src_eos = [] - if new_tgt_bos is not None: - assert len(new_tgt_bos) == len(datasets) - else: - new_tgt_bos = [] - self.src_eos = src_eos - self.tgt_bos = tgt_bos - self.new_src_eos = ( - torch.LongTensor(new_src_eos).cpu() if len(new_src_eos) > 0 else [] - ) - self.new_tgt_bos = ( - torch.LongTensor(new_tgt_bos).cpu() if len(new_tgt_bos) > 0 else [] - ) - self.left_pad_source = self.is_left_pad_source(datasets) - self.left_pad_target = self.is_left_pad_target(datasets) - self.pad_idx = self.src_dict_pad() - - def src_dict_pad(self): - if hasattr(self.datasets[0], "src_dict"): - return self.datasets[0].src_dict.pad() - if hasattr(self.datasets[0], "dataset"): - return self.datasets[0].dataset.src_dict.pad() - raise NotImplementedError("No src_dict is found") - - def __getitem__(self, idx): - dataset_idx, sample_idx = self._get_dataset_and_sample_index(idx) - return dataset_idx, self.datasets[dataset_idx][sample_idx] - - def is_left_pad_source(self, datasets): - def _left_pad_source(ds): - if hasattr(ds, "left_pad_source"): - return ds.left_pad_source - if hasattr(ds, "dataset"): - return _left_pad_source(ds.dataset) - logger.warn(f"{type(ds)} has no left_pad_source, using default True") - return True - - left_pad_source = _left_pad_source(datasets[0]) - for ds in datasets: - if left_pad_source != _left_pad_source(ds): - raise ValueError("Different left_pad_source setting detected!") - return left_pad_source - - def is_left_pad_target(self, datasets): - def _left_pad_target(ds): - if hasattr(ds, "left_pad_target"): - return ds.left_pad_target - if hasattr(ds, "dataset"): - return _left_pad_target(ds.dataset) - logger.warn(f"{type(ds)} has no left_pad_target, using default False") - return False - - left_pad_target = _left_pad_target(datasets[0]) - for ds in datasets: - if left_pad_target != _left_pad_target(ds): - raise ValueError("Different left_pad_target setting detected!") - return left_pad_target - - def collater(self, samples, **extra_args): - if len(samples) == 0: - return samples - - dataset_ids = [s[0] for s in samples] - samples = [s[1] for s in samples] - - if hasattr(self.datasets[0], "collater"): - samples = self.datasets[0].collater(samples, **extra_args) - else: - samples = default_collate(samples, **extra_args) - - if len(self.new_src_eos) > 0: - if self.left_pad_source: - assert ( - samples["net_input"]["src_tokens"][:, -1] != self.src_eos - ).sum() == 0 - samples["net_input"]["src_tokens"][:, -1] = self.new_src_eos[ - dataset_ids - ] - - else: - eos_idx = samples["net_input"]["src_lengths"] - 1 - assert ( - samples["net_input"]["src_tokens"][ - torch.arange(eos_idx.size(0)), eos_idx - ] - != self.src_eos - ).sum() == 0 - samples["net_input"]["src_tokens"].scatter_( - 1, eos_idx.view(-1, 1), self.new_src_eos[dataset_ids].view(-1, 1) - ) - - if len(self.new_tgt_bos) > 0 and "prev_output_tokens" in samples["net_input"]: - if self.left_pad_target: - # TODO: support different padding direction on target side - raise NotImplementedError( - "TransformEosLangPairDataset does not implement --left-pad-target True option" - ) - else: - assert ( - samples["net_input"]["prev_output_tokens"][:, 0] != self.tgt_bos - ).sum() == 0 - samples["net_input"]["prev_output_tokens"][:, 0] = self.new_tgt_bos[ - dataset_ids - ] - - return samples diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/transform_eos_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/transform_eos_dataset.py deleted file mode 100644 index fb14ff018edf13b20f5d0e486692dfb0a37ec6d1..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/transform_eos_dataset.py +++ /dev/null @@ -1,120 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import torch - -from . import FairseqDataset - - -class TransformEosDataset(FairseqDataset): - """A :class:`~fairseq.data.FairseqDataset` wrapper that appends/prepends/strips EOS. - - Note that the transformation is applied in :func:`collater`. - - Args: - dataset (~fairseq.data.FairseqDataset): dataset to wrap - eos (int): index of the end-of-sentence symbol - append_eos_to_src (bool, optional): append EOS to the end of src - remove_eos_from_src (bool, optional): remove EOS from the end of src - append_eos_to_tgt (bool, optional): append EOS to the end of tgt - remove_eos_from_tgt (bool, optional): remove EOS from the end of tgt - """ - - def __init__( - self, - dataset, - eos, - append_eos_to_src=False, - remove_eos_from_src=False, - append_eos_to_tgt=False, - remove_eos_from_tgt=False, - has_target=True, - ): - if not isinstance(dataset, FairseqDataset): - raise ValueError("dataset must be an instance of FairseqDataset") - if append_eos_to_src and remove_eos_from_src: - raise ValueError("cannot combine append_eos_to_src and remove_eos_from_src") - if append_eos_to_tgt and remove_eos_from_tgt: - raise ValueError("cannot combine append_eos_to_tgt and remove_eos_from_tgt") - - self.dataset = dataset - self.eos = torch.LongTensor([eos]) - self.append_eos_to_src = append_eos_to_src - self.remove_eos_from_src = remove_eos_from_src - self.append_eos_to_tgt = append_eos_to_tgt - self.remove_eos_from_tgt = remove_eos_from_tgt - self.has_target = has_target - - # precompute how we should adjust the reported sizes - self._src_delta = 0 - self._src_delta += 1 if append_eos_to_src else 0 - self._src_delta -= 1 if remove_eos_from_src else 0 - self._tgt_delta = 0 - self._tgt_delta += 1 if append_eos_to_tgt else 0 - self._tgt_delta -= 1 if remove_eos_from_tgt else 0 - - self._checked_src = False - self._checked_tgt = False - - def _check_src(self, src, expect_eos): - if not self._checked_src: - assert (src[-1] == self.eos[0]) == expect_eos - self._checked_src = True - - def _check_tgt(self, tgt, expect_eos): - if self.has_target and not self._checked_tgt: - assert (tgt[-1] == self.eos[0]) == expect_eos - self._checked_tgt = True - - def __getitem__(self, index): - return self.dataset[index] - - def __len__(self): - return len(self.dataset) - - def collater(self, samples): - def transform(item): - if self.append_eos_to_src: - self.eos = self.eos.to(device=item["source"].device) - self._check_src(item["source"], expect_eos=False) - item["source"] = torch.cat([item["source"], self.eos]) - if self.remove_eos_from_src: - self.eos = self.eos.to(device=item["source"].device) - self._check_src(item["source"], expect_eos=True) - item["source"] = item["source"][:-1] - if self.append_eos_to_tgt: - self.eos = self.eos.to(device=item["target"].device) - self._check_tgt(item["target"], expect_eos=False) - item["target"] = torch.cat([item["target"], self.eos]) - if self.remove_eos_from_tgt: - self.eos = self.eos.to(device=item["target"].device) - self._check_tgt(item["target"], expect_eos=True) - item["target"] = item["target"][:-1] - return item - - samples = list(map(transform, samples)) - return self.dataset.collater(samples) - - def num_tokens(self, index): - return self.dataset.num_tokens(index) - - def size(self, index): - if self.has_target: - src_len, tgt_len = self.dataset.size(index) - return (src_len + self._src_delta, tgt_len + self._tgt_delta) - else: - return self.dataset.size(index) - - def ordered_indices(self): - # NOTE: we assume that the ordering does not change based on the - # addition or removal of eos - return self.dataset.ordered_indices() - - @property - def supports_prefetch(self): - return getattr(self.dataset, "supports_prefetch", False) - - def prefetch(self, indices): - return self.dataset.prefetch(indices) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/transform_eos_lang_pair_dataset.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/transform_eos_lang_pair_dataset.py deleted file mode 100644 index d8b21090144bfc975d4d5a3ee2c21b2e8acde03d..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/data/transform_eos_lang_pair_dataset.py +++ /dev/null @@ -1,113 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - - -from typing import Optional - -import torch - -from . import FairseqDataset - - -class TransformEosLangPairDataset(FairseqDataset): - """A :class:`~fairseq.data.FairseqDataset` wrapper that transform bos on - collated samples of language pair dataset. - - Note that the transformation is applied in :func:`collater`. - - Args: - dataset (~fairseq.data.FairseqDataset): dataset that collates sample into - LanguagePairDataset schema - src_eos (int): original source end-of-sentence symbol index to be replaced - new_src_eos (int, optional): new end-of-sentence symbol index to replace source eos symbol - tgt_bos (int, optional): original target beginning-of-sentence symbol index to be replaced - new_tgt_bos (int, optional): new beginning-of-sentence symbol index to replace at the - beginning of 'prev_output_tokens' - """ - - def __init__( - self, - dataset: FairseqDataset, - src_eos: int, - new_src_eos: Optional[int] = None, - tgt_bos: Optional[int] = None, - new_tgt_bos: Optional[int] = None, - ): - self.dataset = dataset - self.src_eos = src_eos - self.new_src_eos = new_src_eos - self.tgt_bos = tgt_bos - self.new_tgt_bos = new_tgt_bos - - def __getitem__(self, index): - return self.dataset[index] - - def __len__(self): - return len(self.dataset) - - def collater(self, samples, **extra_args): - samples = self.dataset.collater(samples, **extra_args) - if len(samples) == 0: - return samples - - if "net_input" not in samples: - return samples - - if self.new_src_eos is not None: - if self.dataset.left_pad_source: - assert ( - samples["net_input"]["src_tokens"][:, -1] != self.src_eos - ).sum() == 0 - samples["net_input"]["src_tokens"][:, -1] = self.new_src_eos - else: - eos_idx = samples["net_input"]["src_lengths"] - 1 - assert ( - samples["net_input"]["src_tokens"][ - torch.arange(eos_idx.size(0)), eos_idx - ] - != self.src_eos - ).sum() == 0 - eos_idx = eos_idx.resize_(len(samples["net_input"]["src_lengths"]), 1) - samples["net_input"]["src_tokens"].scatter_( - 1, eos_idx, self.new_src_eos - ) - - if ( - self.new_tgt_bos is not None - and "prev_output_tokens" in samples["net_input"] - ): - if self.dataset.left_pad_target: - # TODO: support different padding direction on target side - raise NotImplementedError( - "TransformEosLangPairDataset does not implement --left-pad-target True option" - ) - else: - assert ( - samples["net_input"]["prev_output_tokens"][:, 0] != self.tgt_bos - ).sum() == 0 - samples["net_input"]["prev_output_tokens"][:, 0] = self.new_tgt_bos - - return samples - - def num_tokens(self, index): - return self.dataset.num_tokens(index) - - def size(self, index): - return self.dataset.size(index) - - @property - def sizes(self): - # dataset.sizes can be a dynamically computed sizes: - return self.dataset.sizes - - def ordered_indices(self): - return self.dataset.ordered_indices() - - @property - def supports_prefetch(self): - return getattr(self.dataset, "supports_prefetch", False) - - def prefetch(self, indices): - return self.dataset.prefetch(indices) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/dataclass/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/dataclass/__init__.py deleted file mode 100644 index 2d241c37c3fb347f3174730214910e432a96943a..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/dataclass/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from .configs import FairseqDataclass -from .constants import ChoiceEnum - -__all__ = [ - "FairseqDataclass", - "ChoiceEnum", -] diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/dataclass/configs.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/dataclass/configs.py deleted file mode 100644 index 4def948a02c8d9bcac8153507159242a9253aa1e..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/dataclass/configs.py +++ /dev/null @@ -1,1123 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import os -import sys -from dataclasses import _MISSING_TYPE, dataclass, field -from typing import Any, List, Optional - -import torch -from fairseq.dataclass.constants import ( - DATASET_IMPL_CHOICES, - DDP_BACKEND_CHOICES, - DDP_COMM_HOOK_CHOICES, - GENERATION_CONSTRAINTS_CHOICES, - GENERATION_DECODING_FORMAT_CHOICES, - LOG_FORMAT_CHOICES, - PIPELINE_CHECKPOINT_CHOICES, - PRINT_ALIGNMENT_CHOICES, - ZERO_SHARDING_CHOICES, -) -from omegaconf import II, MISSING - - -@dataclass -class FairseqDataclass: - """fairseq base dataclass that supported fetching attributes and metas""" - - _name: Optional[str] = None - - @staticmethod - def name(): - return None - - def _get_all_attributes(self) -> List[str]: - return [k for k in self.__dataclass_fields__.keys()] - - def _get_meta( - self, attribute_name: str, meta: str, default: Optional[Any] = None - ) -> Any: - return self.__dataclass_fields__[attribute_name].metadata.get(meta, default) - - def _get_name(self, attribute_name: str) -> str: - return self.__dataclass_fields__[attribute_name].name - - def _get_default(self, attribute_name: str) -> Any: - if hasattr(self, attribute_name): - if str(getattr(self, attribute_name)).startswith("${"): - return str(getattr(self, attribute_name)) - elif str(self.__dataclass_fields__[attribute_name].default).startswith( - "${" - ): - return str(self.__dataclass_fields__[attribute_name].default) - elif ( - getattr(self, attribute_name) - != self.__dataclass_fields__[attribute_name].default - ): - return getattr(self, attribute_name) - - f = self.__dataclass_fields__[attribute_name] - if not isinstance(f.default_factory, _MISSING_TYPE): - return f.default_factory() - return f.default - - def _get_type(self, attribute_name: str) -> Any: - return self.__dataclass_fields__[attribute_name].type - - def _get_help(self, attribute_name: str) -> Any: - return self._get_meta(attribute_name, "help") - - def _get_argparse_const(self, attribute_name: str) -> Any: - return self._get_meta(attribute_name, "argparse_const") - - def _get_argparse_alias(self, attribute_name: str) -> Any: - return self._get_meta(attribute_name, "argparse_alias") - - def _get_choices(self, attribute_name: str) -> Any: - return self._get_meta(attribute_name, "choices") - - @classmethod - def from_namespace(cls, args): - if isinstance(args, cls): - return args - else: - config = cls() - for k in config.__dataclass_fields__.keys(): - if k.startswith("_"): - # private member, skip - continue - if hasattr(args, k): - setattr(config, k, getattr(args, k)) - - return config - - -@dataclass -class CommonConfig(FairseqDataclass): - # This is the core dataclass including common parameters shared by all different jobs. Please append your params to other dataclasses if they were - # used for a particular purpose or task, such as those dedicated for `distributed training`, `optimization`, etc. - no_progress_bar: bool = field( - default=False, metadata={"help": "disable progress bar"} - ) - log_interval: int = field( - default=100, - metadata={ - "help": "log progress every N batches (when progress bar is disabled)" - }, - ) - log_format: Optional[LOG_FORMAT_CHOICES] = field( - default=None, metadata={"help": "log format to use"} - ) - log_file: Optional[str] = field( - default=None, metadata={"help": "log file to copy metrics to."} - ) - aim_repo: Optional[str] = field( - default=None, - metadata={"help": "path to Aim repository"}, - ) - aim_run_hash: Optional[str] = field( - default=None, - metadata={ - "help": "Aim run hash. If skipped, creates or continues run " - "based on save_dir" - }, - ) - tensorboard_logdir: Optional[str] = field( - default=None, - metadata={ - "help": "path to save logs for tensorboard, should match --logdir " - "of running tensorboard (default: no tensorboard logging)" - }, - ) - wandb_project: Optional[str] = field( - default=None, - metadata={"help": "Weights and Biases project name to use for logging"}, - ) - azureml_logging: Optional[bool] = field( - default=False, - metadata={"help": "Log scalars to AzureML context"}, - ) - seed: int = field( - default=1, metadata={"help": "pseudo random number generator seed"} - ) - cpu: bool = field(default=False, metadata={"help": "use CPU instead of CUDA"}) - tpu: bool = field(default=False, metadata={"help": "use TPU instead of CUDA"}) - bf16: bool = field(default=False, metadata={"help": "use bfloat16; implies --tpu"}) - memory_efficient_bf16: bool = field( - default=False, - metadata={ - "help": "use a memory-efficient version of BF16 training; implies --bf16" - }, - ) - fp16: bool = field(default=False, metadata={"help": "use FP16"}) - memory_efficient_fp16: bool = field( - default=False, - metadata={ - "help": "use a memory-efficient version of FP16 training; implies --fp16" - }, - ) - fp16_no_flatten_grads: bool = field( - default=False, metadata={"help": "don't flatten FP16 grads tensor"} - ) - fp16_init_scale: int = field( - default=2**7, metadata={"help": "default FP16 loss scale"} - ) - fp16_scale_window: Optional[int] = field( - default=None, - metadata={"help": "number of updates before increasing loss scale"}, - ) - fp16_scale_tolerance: float = field( - default=0.0, - metadata={ - "help": "pct of updates that can overflow before decreasing the loss scale" - }, - ) - on_cpu_convert_precision: bool = field( - default=False, - metadata={ - "help": "if set, the floating point conversion to fp16/bf16 runs on CPU. " - "This reduces bus transfer time and GPU memory usage." - }, - ) - min_loss_scale: float = field( - default=1e-4, - metadata={ - "help": "minimum FP16/AMP loss scale, after which training is stopped" - }, - ) - threshold_loss_scale: Optional[float] = field( - default=None, metadata={"help": "threshold FP16 loss scale from below"} - ) - amp: bool = field(default=False, metadata={"help": "use automatic mixed precision"}) - amp_batch_retries: int = field( - default=2, - metadata={ - "help": "number of retries of same batch after reducing loss scale with AMP" - }, - ) - amp_init_scale: int = field( - default=2**7, metadata={"help": "default AMP loss scale"} - ) - amp_scale_window: Optional[int] = field( - default=None, - metadata={"help": "number of updates before increasing AMP loss scale"}, - ) - user_dir: Optional[str] = field( - default=None, - metadata={ - "help": "path to a python module containing custom extensions (tasks and/or architectures)" - }, - ) - empty_cache_freq: int = field( - default=0, - metadata={"help": "how often to clear the PyTorch CUDA cache (0 to disable)"}, - ) - all_gather_list_size: int = field( - default=16384, - metadata={"help": "number of bytes reserved for gathering stats from workers"}, - ) - model_parallel_size: int = field( - default=1, metadata={"help": "total number of GPUs to parallelize model over"} - ) - quantization_config_path: Optional[str] = field( - default=None, metadata={"help": "path to quantization config file"} - ) - profile: bool = field( - default=False, metadata={"help": "enable autograd profiler emit_nvtx"} - ) - reset_logging: bool = field( - default=False, - metadata={ - "help": "when using Hydra, reset the logging at the beginning of training" - }, - ) - suppress_crashes: bool = field( - default=False, - metadata={ - "help": "suppress crashes when training with the hydra_train entry point so that the " - "main method can return a value (useful for sweeps)" - }, - ) - use_plasma_view: bool = field( - default=False, metadata={"help": "Store indices and sizes in shared memory"} - ) - plasma_path: Optional[str] = field( - default="/tmp/plasma", - metadata={ - "help": "path to run plasma_store, defaults to /tmp/plasma. Paths outside /tmp tend to fail." - }, - ) - - -@dataclass -class DistributedTrainingConfig(FairseqDataclass): - distributed_world_size: int = field( - default=max(1, torch.cuda.device_count()), - metadata={ - "help": "total number of GPUs across all nodes (default: all visible GPUs)" - }, - ) - distributed_num_procs: Optional[int] = field( - default=max(1, torch.cuda.device_count()), - metadata={ - "help": "total number of processes to fork (default: all visible GPUs)" - }, - ) - distributed_rank: Optional[int] = field( - default=0, metadata={"help": "rank of the current worker"} - ) - distributed_backend: str = field( - default="nccl", metadata={"help": "distributed backend"} - ) - distributed_init_method: Optional[str] = field( - default=None, - metadata={ - "help": "typically tcp://hostname:port that will be used to " - "establish initial connetion" - }, - ) - distributed_port: int = field( - default=-1, - metadata={ - "help": "port number (not required if using --distributed-init-method)" - }, - ) - device_id: int = field( - default=os.getenv("LOCAL_RANK", 0), - metadata={ - "help": "which GPU to use (by default looks for $LOCAL_RANK, usually configured automatically)", - "argparse_alias": "--local_rank", - }, - ) - distributed_no_spawn: bool = field( - default=False, - metadata={ - "help": "do not spawn multiple processes even if multiple GPUs are visible" - }, - ) - ddp_backend: DDP_BACKEND_CHOICES = field( - default="pytorch_ddp", metadata={"help": "DistributedDataParallel backend"} - ) - ddp_comm_hook: DDP_COMM_HOOK_CHOICES = field( - default="none", metadata={"help": "communication hook"} - ) - bucket_cap_mb: int = field( - default=25, metadata={"help": "bucket size for reduction"} - ) - fix_batches_to_gpus: bool = field( - default=False, - metadata={ - "help": "don't shuffle batches between GPUs; this reduces overall " - "randomness and may affect precision but avoids the cost of re-reading the data" - }, - ) - find_unused_parameters: bool = field( - default=False, - metadata={ - "help": "disable unused parameter detection (not applicable to " - "--ddp-backend=legacy_ddp)" - }, - ) - gradient_as_bucket_view: bool = field( - default=False, - metadata={ - "help": "when set to True, gradients will be views pointing to different offsets of allreduce communication buckets. This can reduce peak memory usage, where the saved memory size will be equal to the total gradients size. " - "--gradient-as-bucket-view=gradient_as_bucket_view)" - }, - ) - fast_stat_sync: bool = field( - default=False, - metadata={"help": "[deprecated] this is now defined per Criterion"}, - ) - heartbeat_timeout: int = field( - default=-1, - metadata={ - "help": "kill the job if no progress is made in N seconds; " - "set to -1 to disable" - }, - ) - broadcast_buffers: bool = field( - default=False, - metadata={ - "help": "Copy non-trainable parameters between GPUs, such as " - "batchnorm population statistics" - }, - ) - slowmo_momentum: Optional[float] = field( - default=None, - metadata={ - "help": "SlowMo momentum term; by default use 0.0 for 16 GPUs, " - "0.2 for 32 GPUs; 0.5 for 64 GPUs, 0.6 for > 64 GPUs" - }, - ) - slowmo_base_algorithm: str = field( - default="localsgd", - metadata={ - "help": "Base algorithm. Either 'localsgd' or 'sgp'. Please refer " - "to the documentation of 'slowmo_base_algorithm' parameter in " - "https://fairscale.readthedocs.io/en/latest/api/experimental/nn/slowmo_ddp.html " - "for more details" - }, - ) - localsgd_frequency: int = field( - default=3, metadata={"help": "Local SGD allreduce frequency"} - ) - nprocs_per_node: int = field( - default=max(1, torch.cuda.device_count()), - metadata={ - "help": "number of GPUs in each node. An allreduce operation across GPUs in " - "a node is very fast. Hence, we do allreduce across GPUs in a node, " - "and gossip across different nodes" - }, - ) - pipeline_model_parallel: bool = field( - default=False, - metadata={"help": "if set, use pipeline model parallelism across GPUs"}, - ) - pipeline_balance: Optional[str] = field( - default=None, - metadata={ - "help": "partition the model into N_K pieces, where each piece " - "contains N_i layers. The sum(args.pipeline_balance) " - "should equal the total number of layers in the model" - }, - ) - pipeline_devices: Optional[str] = field( - default=None, - metadata={ - "help": "a list of device indices indicating which device to place " - "each of the N_K partitions. The length of this list should " - "equal the length of the --pipeline-balance argument" - }, - ) - pipeline_chunks: Optional[int] = field( - default=0, metadata={"help": "microbatch count for pipeline model parallelism"} - ) - pipeline_encoder_balance: Optional[str] = field( - default=None, - metadata={ - "help": "partition the pipeline parallel encoder into N_K pieces, where each piece " - "contains N_i layers. The sum(args.pipeline_encoder_balance) " - "should equal the total number of encoder layers in the model" - }, - ) - pipeline_encoder_devices: Optional[str] = field( - default=None, - metadata={ - "help": "a list of device indices indicating which device to place " - "each of the N_K partitions. The length of this list should " - "equal the length of the --pipeline-encoder-balance argument" - }, - ) - pipeline_decoder_balance: Optional[str] = field( - default=None, - metadata={ - "help": "partition the pipeline parallel decoder into N_K pieces, where each piece " - "contains N_i layers. The sum(args.pipeline_decoder_balance) " - "should equal the total number of decoder layers in the model" - }, - ) - pipeline_decoder_devices: Optional[str] = field( - default=None, - metadata={ - "help": "a list of device indices indicating which device to place " - "each of the N_K partitions. The length of this list should " - "equal the length of the --pipeline-decoder-balance argument" - }, - ) - pipeline_checkpoint: PIPELINE_CHECKPOINT_CHOICES = field( - default="never", - metadata={"help": "checkpointing mode for pipeline model parallelism"}, - ) - zero_sharding: ZERO_SHARDING_CHOICES = field( - default="none", metadata={"help": "ZeRO sharding"} - ) - fp16: bool = II("common.fp16") - memory_efficient_fp16: bool = II("common.memory_efficient_fp16") - tpu: bool = II("common.tpu") - # configuration for --ddp-backend=fully_sharded - no_reshard_after_forward: bool = field( - default=False, - metadata={"help": "don't reshard parameters after forward pass"}, - ) - fp32_reduce_scatter: bool = field( - default=False, - metadata={"help": "reduce-scatter grads in FP32"}, - ) - cpu_offload: bool = field( - default=False, metadata={"help": "offload FP32 params to CPU"} - ) - use_sharded_state: bool = field( - default=False, - metadata={"help": "use sharded checkpoint files"}, - ) - not_fsdp_flatten_parameters: bool = field( - default=False, - metadata={"help": "not flatten parameter param for fsdp"}, - ) - - -@dataclass -class DatasetConfig(FairseqDataclass): - num_workers: int = field( - default=1, metadata={"help": "how many subprocesses to use for data loading"} - ) - skip_invalid_size_inputs_valid_test: bool = field( - default=False, - metadata={"help": "ignore too long or too short lines in valid and test set"}, - ) - max_tokens: Optional[int] = field( - default=None, metadata={"help": "maximum number of tokens in a batch"} - ) - batch_size: Optional[int] = field( - default=None, - metadata={ - "help": "number of examples in a batch", - "argparse_alias": "--max-sentences", - }, - ) - required_batch_size_multiple: int = field( - default=8, metadata={"help": "batch size will be a multiplier of this value"} - ) - required_seq_len_multiple: int = field( - default=1, - metadata={ - "help": "maximum sequence length in batch will be a multiplier of this value" - }, - ) - dataset_impl: Optional[DATASET_IMPL_CHOICES] = field( - default=None, metadata={"help": "output dataset implementation"} - ) - data_buffer_size: int = field( - default=10, metadata={"help": "Number of batches to preload"} - ) - train_subset: str = field( - default="train", - metadata={"help": "data subset to use for training (e.g. train, valid, test)"}, - ) - valid_subset: str = field( - default="valid", - metadata={ - "help": "comma separated list of data subsets to use for validation" - " (e.g. train, valid, test)" - }, - ) - combine_valid_subsets: Optional[bool] = field( - default=None, - metadata={ - "help": "comma separated list of data subsets to use for validation" - " (e.g. train, valid, test)", - "argparse_alias": "--combine-val", - }, - ) - ignore_unused_valid_subsets: Optional[bool] = field( - default=False, - metadata={"help": "do not raise error if valid subsets are ignored"}, - ) - - validate_interval: int = field( - default=1, metadata={"help": "validate every N epochs"} - ) - validate_interval_updates: int = field( - default=0, metadata={"help": "validate every N updates"} - ) - validate_after_updates: int = field( - default=0, metadata={"help": "dont validate until reaching this many updates"} - ) - fixed_validation_seed: Optional[int] = field( - default=None, metadata={"help": "specified random seed for validation"} - ) - disable_validation: bool = field( - default=False, metadata={"help": "disable validation"} - ) - max_tokens_valid: Optional[int] = field( - default=II("dataset.max_tokens"), - metadata={ - "help": "maximum number of tokens in a validation batch" - " (defaults to --max-tokens)" - }, - ) - batch_size_valid: Optional[int] = field( - default=II("dataset.batch_size"), - metadata={ - "help": "batch size of the validation batch (defaults to --batch-size)", - "argparse_alias": "--max-sentences-valid", - }, - ) - max_valid_steps: Optional[int] = field( - default=None, - metadata={"help": "How many batches to evaluate", "argparse_alias": "--nval"}, - ) - curriculum: int = field( - default=0, metadata={"help": "don't shuffle batches for first N epochs"} - ) - gen_subset: str = field( - default="test", - metadata={"help": "data subset to generate (train, valid, test)"}, - ) - num_shards: int = field( - default=1, metadata={"help": "shard generation over N shards"} - ) - shard_id: int = field( - default=0, metadata={"help": "id of the shard to generate (id < num_shards)"} - ) - grouped_shuffling: bool = field( - default=False, - metadata={ - "help": "shuffle batches in groups of num_shards to enable similar sequence lengths on each GPU worker when batches are sorted by length", - }, - ) - update_epoch_batch_itr: bool = field( - default=II("dataset.grouped_shuffling"), - metadata={ - "help": "if true then prevents the reuse the epoch batch iterator by setting can_reuse_epoch_itr to false, defaults to --grouped-shuffling )", - }, - ) - update_ordered_indices_seed: bool = field( - default=False, - metadata={ - "help": "if true then increment seed with epoch for getting batch iterators, defautls to False.", - }, - ) - - -@dataclass -class OptimizationConfig(FairseqDataclass): - max_epoch: int = field( - default=0, metadata={"help": "force stop training at specified epoch"} - ) - max_update: int = field( - default=0, metadata={"help": "force stop training at specified update"} - ) - stop_time_hours: float = field( - default=0, - metadata={ - "help": "force stop training after specified cumulative time (if >0)" - }, - ) - clip_norm: float = field( - default=0.0, metadata={"help": "clip threshold of gradients"} - ) - sentence_avg: bool = field( - default=False, - metadata={ - "help": "normalize gradients by the number of sentences in a batch" - " (default is to normalize by number of tokens)" - }, - ) - update_freq: List[int] = field( - default_factory=lambda: [1], - metadata={"help": "update parameters every N_i batches, when in epoch i"}, - ) - lr: List[float] = field( - default_factory=lambda: [0.25], - metadata={ - "help": "learning rate for the first N epochs; all epochs >N using LR_N" - " (note: this may be interpreted differently depending on --lr-scheduler)" - }, - ) - stop_min_lr: float = field( - default=-1.0, - metadata={"help": "stop training when the learning rate reaches this minimum"}, - ) - use_bmuf: bool = field( - default=False, - metadata={ - "help": "specify global optimizer for syncing models on different GPUs/shards" - }, - ) - skip_remainder_batch: Optional[bool] = field( - default=False, - metadata={ - "help": "if set, include the last (partial) batch of each epoch in training" - " (default is to skip it)." - }, - ) - - -@dataclass -class CheckpointConfig(FairseqDataclass): - save_dir: str = field( - default="checkpoints", metadata={"help": "path to save checkpoints"} - ) - restore_file: str = field( - default="checkpoint_last.pt", - metadata={ - "help": "filename from which to load checkpoint " - "(default: <save-dir>/checkpoint_last.pt" - }, - ) - continue_once: Optional[str] = field( - default=None, - metadata={ - "help": "continues from this checkpoint, unless a checkpoint indicated in 'restore_file' option is present" - }, - ) - finetune_from_model: Optional[str] = field( - default=None, - metadata={ - "help": "finetune from a pretrained model; note that meters and lr scheduler will be reset" - }, - ) - reset_dataloader: bool = field( - default=False, - metadata={ - "help": "if set, does not reload dataloader state from the checkpoint" - }, - ) - reset_lr_scheduler: bool = field( - default=False, - metadata={ - "help": "if set, does not load lr scheduler state from the checkpoint" - }, - ) - reset_meters: bool = field( - default=False, - metadata={"help": "if set, does not load meters from the checkpoint"}, - ) - reset_optimizer: bool = field( - default=False, - metadata={"help": "if set, does not load optimizer state from the checkpoint"}, - ) - optimizer_overrides: str = field( - default="{}", - metadata={ - "help": "a dictionary used to override optimizer args when loading a checkpoint" - }, - ) - save_interval: int = field( - default=1, metadata={"help": "save a checkpoint every N epochs"} - ) - save_interval_updates: int = field( - default=0, metadata={"help": "save a checkpoint (and validate) every N updates"} - ) - keep_interval_updates: int = field( - default=-1, - metadata={ - "help": "keep the last N checkpoints saved with --save-interval-updates" - }, - ) - keep_interval_updates_pattern: int = field( - default=-1, - metadata={ - "help": "when used with --keep-interval-updates, skips deleting " - "any checkpoints with update X where " - "X %% keep_interval_updates_pattern == 0" - }, - ) - keep_last_epochs: int = field( - default=-1, metadata={"help": "keep last N epoch checkpoints"} - ) - keep_best_checkpoints: int = field( - default=-1, metadata={"help": "keep best N checkpoints based on scores"} - ) - no_save: bool = field( - default=False, metadata={"help": "don't save models or checkpoints"} - ) - no_epoch_checkpoints: bool = field( - default=False, metadata={"help": "only store last and best checkpoints"} - ) - no_last_checkpoints: bool = field( - default=False, metadata={"help": "don't store last checkpoints"} - ) - no_save_optimizer_state: bool = field( - default=False, - metadata={"help": "don't save optimizer-state as part of checkpoint"}, - ) - best_checkpoint_metric: str = field( - default="loss", metadata={"help": 'metric to use for saving "best" checkpoints'} - ) - maximize_best_checkpoint_metric: bool = field( - default=False, - metadata={ - "help": 'select the largest metric value for saving "best" checkpoints' - }, - ) - patience: int = field( - default=-1, - metadata={ - "help": ( - "early stop training if valid performance doesn't " - "improve for N consecutive validation runs; note " - "that this is influenced by --validate-interval" - ) - }, - ) - checkpoint_suffix: str = field( - default="", metadata={"help": "suffix to add to the checkpoint file name"} - ) - checkpoint_shard_count: int = field( - default=1, - metadata={ - "help": "Number of shards containing the checkpoint - " - "if the checkpoint is over 300GB, it is preferable " - "to split it into shards to prevent OOM on CPU while loading " - "the checkpoint" - }, - ) - load_checkpoint_on_all_dp_ranks: bool = field( - default=False, - metadata={ - "help": "load checkpoints on all data parallel devices " - "(default: only load on rank 0 and broadcast to other devices)" - }, - ) - write_checkpoints_asynchronously: bool = field( - default=False, - metadata={ - "help": ( - "Write checkpoints asynchronously in a separate " - "thread. NOTE: This feature is currently being tested." - ), - "argparse_alias": "--save-async", - }, - ) - model_parallel_size: int = II("common.model_parallel_size") - - -@dataclass -class FairseqBMUFConfig(FairseqDataclass): - block_lr: float = field( - default=1, metadata={"help": "block learning rate for bmuf"} - ) - block_momentum: float = field( - default=0.875, metadata={"help": "block momentum for bmuf"} - ) - global_sync_iter: int = field( - default=50, metadata={"help": "Iteration for syncing global model"} - ) - warmup_iterations: int = field( - default=500, metadata={"help": "warmup iterations for model to broadcast"} - ) - use_nbm: bool = field( - default=False, - metadata={"help": "Specify whether you want to use classical BM / Nesterov BM"}, - ) - average_sync: bool = field( - default=False, - metadata={ - "help": "Specify whether you want to average the local momentum after each sync" - }, - ) - distributed_world_size: int = II("distributed_training.distributed_world_size") - - -@dataclass -class GenerationConfig(FairseqDataclass): - beam: int = field( - default=5, - metadata={"help": "beam size"}, - ) - nbest: int = field( - default=1, - metadata={"help": "number of hypotheses to output"}, - ) - max_len_a: float = field( - default=0, - metadata={ - "help": "generate sequences of maximum length ax + b, where x is the source length" - }, - ) - max_len_b: int = field( - default=200, - metadata={ - "help": "generate sequences of maximum length ax + b, where x is the source length" - }, - ) - min_len: int = field( - default=1, - metadata={"help": "minimum generation length"}, - ) - match_source_len: bool = field( - default=False, - metadata={"help": "generations should match the source length"}, - ) - unnormalized: bool = field( - default=False, - metadata={"help": "compare unnormalized hypothesis scores"}, - ) - no_early_stop: bool = field( - default=False, - metadata={"help": "deprecated"}, - ) - no_beamable_mm: bool = field( - default=False, - metadata={"help": "don't use BeamableMM in attention layers"}, - ) - lenpen: float = field( - default=1, - metadata={ - "help": "length penalty: <1.0 favors shorter, >1.0 favors longer sentences" - }, - ) - unkpen: float = field( - default=0, - metadata={ - "help": "unknown word penalty: <0 produces more unks, >0 produces fewer" - }, - ) - replace_unk: Optional[str] = field( - default=None, - metadata={ - "help": "perform unknown replacement (optionally with alignment dictionary)", - "argparse_const": "@@ ", - }, - ) - sacrebleu: bool = field( - default=False, - metadata={"help": "score with sacrebleu"}, - ) - score_reference: bool = field( - default=False, - metadata={"help": "just score the reference translation"}, - ) - prefix_size: int = field( - default=0, - metadata={"help": "initialize generation by target prefix of given length"}, - ) - no_repeat_ngram_size: int = field( - default=0, - metadata={ - "help": "ngram blocking such that this size ngram cannot be repeated in the generation" - }, - ) - sampling: bool = field( - default=False, - metadata={"help": "sample hypotheses instead of using beam search"}, - ) - sampling_topk: int = field( - default=-1, - metadata={"help": "sample from top K likely next words instead of all words"}, - ) - sampling_topp: float = field( - default=-1.0, - metadata={ - "help": "sample from the smallest set whose cumulative probability mass exceeds p for next words" - }, - ) - constraints: Optional[GENERATION_CONSTRAINTS_CHOICES] = field( - default=None, - metadata={ - "help": "enables lexically constrained decoding", - "argparse_const": "ordered", - }, - ) - temperature: float = field( - default=1.0, - metadata={"help": "temperature for generation"}, - ) - diverse_beam_groups: int = field( - default=-1, - metadata={"help": "number of groups for Diverse Beam Search"}, - ) - diverse_beam_strength: float = field( - default=0.5, - metadata={"help": "strength of diversity penalty for Diverse Beam Search"}, - ) - diversity_rate: float = field( - default=-1.0, - metadata={"help": "strength of diversity penalty for Diverse Siblings Search"}, - ) - print_alignment: Optional[PRINT_ALIGNMENT_CHOICES] = field( - default=None, - metadata={ - "help": "if set, uses attention feedback to compute and print alignment to source tokens " - "(valid options are: hard, soft, otherwise treated as hard alignment)", - "argparse_const": "hard", - }, - ) - print_step: bool = field( - default=False, - metadata={"help": "print steps"}, - ) - lm_path: Optional[str] = field( - default=None, - metadata={"help": "path to lm checkpoint for lm fusion"}, - ) - lm_weight: float = field( - default=0.0, - metadata={"help": "weight for lm probs for lm fusion"}, - ) - - # arguments for iterative refinement generator - iter_decode_eos_penalty: float = field( - default=0.0, - metadata={"help": "if > 0.0, it penalized early-stopping in decoding."}, - ) - iter_decode_max_iter: int = field( - default=10, - metadata={"help": "maximum iterations for iterative refinement."}, - ) - iter_decode_force_max_iter: bool = field( - default=False, - metadata={ - "help": "if set, run exact the maximum number of iterations without early stop" - }, - ) - iter_decode_with_beam: int = field( - default=1, - metadata={ - "help": "if > 1, model will generate translations varying by the lengths." - }, - ) - iter_decode_with_external_reranker: bool = field( - default=False, - metadata={ - "help": "if set, the last checkpoint are assumed to be a reranker to rescore the translations" - }, - ) - retain_iter_history: bool = field( - default=False, - metadata={ - "help": "if set, decoding returns the whole history of iterative refinement" - }, - ) - retain_dropout: bool = field( - default=False, - metadata={"help": "Use dropout at inference time"}, - ) - # temporarily set to Any until https://github.com/facebookresearch/hydra/issues/1117 is fixed - # retain_dropout_modules: Optional[List[str]] = field( - retain_dropout_modules: Any = field( - default=None, - metadata={ - "help": "if set, only retain dropout for the specified modules; " - "if not set, then dropout will be retained for all modules" - }, - ) - # special decoding format for advanced decoding. - decoding_format: Optional[GENERATION_DECODING_FORMAT_CHOICES] = field( - default=None, - metadata={"help": "special decoding format for advanced decoding."}, - ) - no_seed_provided: bool = field( - default=False, - metadata={"help": "if set, dont use seed for initializing random generators"}, - ) - eos_token: Optional[str] = field( - default=None, - metadata={"help": "EOS token"}, - ) - - -@dataclass -class CommonEvalConfig(FairseqDataclass): - path: Optional[str] = field( - default=None, - metadata={"help": "path(s) to model file(s), colon separated"}, - ) - post_process: Optional[str] = field( - default=None, - metadata={ - "help": ( - "post-process text by removing BPE, letter segmentation, etc. " - "Valid options can be found in fairseq.data.utils.post_process." - ), - "argparse_const": "subword_nmt", - "argparse_alias": "--remove-bpe", - }, - ) - quiet: bool = field(default=False, metadata={"help": "only print final scores"}) - model_overrides: str = field( - default="{}", - metadata={ - "help": "a dictionary used to override model args at generation that were used during model training" - }, - ) - results_path: Optional[str] = field( - default=None, metadata={"help": "path to save eval results (optional)"} - ) - - -@dataclass -class EvalLMConfig(FairseqDataclass): - output_word_probs: bool = field( - default=False, - metadata={ - "help": "if set, outputs words and their predicted log probabilities to standard output" - }, - ) - output_word_stats: bool = field( - default=False, - metadata={ - "help": "if set, outputs word statistics such as word count, average probability, etc" - }, - ) - context_window: int = field( - default=0, - metadata={ - "help": "ensures that every evaluated token has access to a context of at least this size, if possible" - }, - ) - softmax_batch: int = field( - default=sys.maxsize, - metadata={ - "help": "if BxT is more than this, will batch the softmax over vocab to this amount of tokens, in order to fit into GPU memory" - }, - ) - - -@dataclass -class InteractiveConfig(FairseqDataclass): - buffer_size: int = field( - default=0, - metadata={ - "help": "read this many sentences into a buffer before processing them" - }, - ) - input: str = field( - default="-", - metadata={"help": "file to read from; use - for stdin"}, - ) - - -@dataclass -class EMAConfig(FairseqDataclass): - store_ema: bool = field( - default=False, metadata={help: "store exponential moving average shadow model"} - ) - ema_decay: float = field( - default=0.9999, metadata={"help": "decay for exponential moving average model"} - ) - ema_start_update: int = field( - default=0, metadata={"help": "start EMA update after this many model updates"} - ) - ema_seed_model: Optional[str] = field( - default=None, - metadata={ - "help": "Seed to load EMA model from. " - "Used to load EMA model separately from the actual model." - }, - ) - ema_update_freq: int = field( - default=1, metadata={"help": "Do EMA update every this many model updates"} - ) - ema_fp32: bool = field( - default=False, - metadata={"help": "If true, store EMA model in fp32 even if model is in fp16"}, - ) - - -@dataclass -class FairseqConfig(FairseqDataclass): - common: CommonConfig = CommonConfig() - common_eval: CommonEvalConfig = CommonEvalConfig() - distributed_training: DistributedTrainingConfig = DistributedTrainingConfig() - dataset: DatasetConfig = DatasetConfig() - optimization: OptimizationConfig = OptimizationConfig() - checkpoint: CheckpointConfig = CheckpointConfig() - bmuf: FairseqBMUFConfig = FairseqBMUFConfig() - generation: GenerationConfig = GenerationConfig() - eval_lm: EvalLMConfig = EvalLMConfig() - interactive: InteractiveConfig = InteractiveConfig() - model: Any = MISSING - task: Any = None - criterion: Any = None - optimizer: Any = None - lr_scheduler: Any = None - scoring: Any = None - bpe: Any = None - tokenizer: Any = None - ema: EMAConfig = EMAConfig() diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/dataclass/constants.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/dataclass/constants.py deleted file mode 100644 index 5af92f2b3aa51e460f0b045a348d3766f93eb90b..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/dataclass/constants.py +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from enum import Enum, EnumMeta -from typing import List - - -class StrEnumMeta(EnumMeta): - # this is workaround for submitit pickling leading to instance checks failing in hydra for StrEnum, see - # https://github.com/facebookresearch/hydra/issues/1156 - @classmethod - def __instancecheck__(cls, other): - return "enum" in str(type(other)) - - -class StrEnum(Enum, metaclass=StrEnumMeta): - def __str__(self): - return self.value - - def __eq__(self, other: str): - return self.value == other - - def __repr__(self): - return self.value - - def __hash__(self): - return hash(str(self)) - - -def ChoiceEnum(choices: List[str]): - """return the Enum class used to enforce list of choices""" - return StrEnum("Choices", {k: k for k in choices}) - - -LOG_FORMAT_CHOICES = ChoiceEnum(["json", "none", "simple", "tqdm"]) -DDP_BACKEND_CHOICES = ChoiceEnum( - [ - "c10d", # alias for pytorch_ddp - "fully_sharded", # FullyShardedDataParallel from fairscale - "legacy_ddp", - "no_c10d", # alias for legacy_ddp - "pytorch_ddp", - "slowmo", - ] -) -DDP_COMM_HOOK_CHOICES = ChoiceEnum(["none", "fp16"]) -DATASET_IMPL_CHOICES = ChoiceEnum(["raw", "lazy", "cached", "mmap", "fasta", "huffman"]) -GENERATION_CONSTRAINTS_CHOICES = ChoiceEnum(["ordered", "unordered"]) -GENERATION_DECODING_FORMAT_CHOICES = ChoiceEnum( - ["unigram", "ensemble", "vote", "dp", "bs"] -) -ZERO_SHARDING_CHOICES = ChoiceEnum(["none", "os"]) -PIPELINE_CHECKPOINT_CHOICES = ChoiceEnum(["always", "never", "except_last"]) -PRINT_ALIGNMENT_CHOICES = ChoiceEnum(["hard", "soft"]) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/dataclass/initialize.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/dataclass/initialize.py deleted file mode 100644 index 5a7784bad194761b6d60ccfa5aed2fc01aa123c0..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/dataclass/initialize.py +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. -"""isort:skip_file""" - -import logging -from hydra.core.config_store import ConfigStore -from fairseq.dataclass.configs import FairseqConfig -from omegaconf import DictConfig, OmegaConf - - -logger = logging.getLogger(__name__) - - -def hydra_init(cfg_name="config") -> None: - - cs = ConfigStore.instance() - cs.store(name=f"{cfg_name}", node=FairseqConfig) - - for k in FairseqConfig.__dataclass_fields__: - v = FairseqConfig.__dataclass_fields__[k].default - try: - cs.store(name=k, node=v) - except BaseException: - logger.error(f"{k} - {v}") - raise - - -def add_defaults(cfg: DictConfig) -> None: - """This function adds default values that are stored in dataclasses that hydra doesn't know about""" - - from fairseq.registry import REGISTRIES - from fairseq.tasks import TASK_DATACLASS_REGISTRY - from fairseq.models import ARCH_MODEL_NAME_REGISTRY, MODEL_DATACLASS_REGISTRY - from fairseq.dataclass.utils import merge_with_parent - from typing import Any - - OmegaConf.set_struct(cfg, False) - - for k, v in FairseqConfig.__dataclass_fields__.items(): - field_cfg = cfg.get(k) - if field_cfg is not None and v.type == Any: - dc = None - - if isinstance(field_cfg, str): - field_cfg = DictConfig({"_name": field_cfg}) - field_cfg.__dict__["_parent"] = field_cfg.__dict__["_parent"] - - name = getattr(field_cfg, "_name", None) - - if k == "task": - dc = TASK_DATACLASS_REGISTRY.get(name) - elif k == "model": - name = ARCH_MODEL_NAME_REGISTRY.get(name, name) - dc = MODEL_DATACLASS_REGISTRY.get(name) - elif k in REGISTRIES: - dc = REGISTRIES[k]["dataclass_registry"].get(name) - - if dc is not None: - cfg[k] = merge_with_parent(dc, field_cfg) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/dataclass/utils.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/dataclass/utils.py deleted file mode 100644 index 18f550c47c5639db0803e9c16d2f0202e1768110..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/dataclass/utils.py +++ /dev/null @@ -1,493 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import ast -import inspect -import logging -import os -import re -from argparse import ArgumentError, ArgumentParser, Namespace -from dataclasses import _MISSING_TYPE, MISSING, is_dataclass -from enum import Enum -from typing import Any, Dict, List, Optional, Tuple, Type - -from fairseq.dataclass import FairseqDataclass -from fairseq.dataclass.configs import FairseqConfig -from hydra.core.global_hydra import GlobalHydra -from hydra.experimental import compose, initialize -from omegaconf import DictConfig, OmegaConf, _utils, open_dict - -logger = logging.getLogger(__name__) - - -def eval_str_list(x, x_type=float): - if x is None: - return None - if isinstance(x, str): - if len(x) == 0: - return [] - x = ast.literal_eval(x) - try: - return list(map(x_type, x)) - except TypeError: - return [x_type(x)] - - -def interpret_dc_type(field_type): - if isinstance(field_type, str): - raise RuntimeError("field should be a type") - - if field_type == Any: - return str - - typestring = str(field_type) - if re.match( - r"(typing.|^)Union\[(.*), NoneType\]$", typestring - ) or typestring.startswith("typing.Optional"): - return field_type.__args__[0] - return field_type - - -def gen_parser_from_dataclass( - parser: ArgumentParser, - dataclass_instance: FairseqDataclass, - delete_default: bool = False, - with_prefix: Optional[str] = None, -) -> None: - """ - convert a dataclass instance to tailing parser arguments. - - If `with_prefix` is provided, prefix all the keys in the resulting parser with it. It means that we are - building a flat namespace from a structured dataclass (see transformer_config.py for example). - """ - - def argparse_name(name: str): - if name == "data" and (with_prefix is None or with_prefix == ""): - # normally data is positional args, so we don't add the -- nor the prefix - return name - if name == "_name": - # private member, skip - return None - full_name = "--" + name.replace("_", "-") - if with_prefix is not None and with_prefix != "": - # if a prefix is specified, construct the prefixed arg name - full_name = with_prefix + "-" + full_name[2:] # strip -- when composing - return full_name - - def get_kwargs_from_dc( - dataclass_instance: FairseqDataclass, k: str - ) -> Dict[str, Any]: - """k: dataclass attributes""" - - kwargs = {} - - field_type = dataclass_instance._get_type(k) - inter_type = interpret_dc_type(field_type) - - field_default = dataclass_instance._get_default(k) - - if isinstance(inter_type, type) and issubclass(inter_type, Enum): - field_choices = [t.value for t in list(inter_type)] - else: - field_choices = None - - field_help = dataclass_instance._get_help(k) - field_const = dataclass_instance._get_argparse_const(k) - - if isinstance(field_default, str) and field_default.startswith("${"): - kwargs["default"] = field_default - else: - if field_default is MISSING: - kwargs["required"] = True - if field_choices is not None: - kwargs["choices"] = field_choices - if ( - isinstance(inter_type, type) - and (issubclass(inter_type, List) or issubclass(inter_type, Tuple)) - ) or ("List" in str(inter_type) or "Tuple" in str(inter_type)): - if "int" in str(inter_type): - kwargs["type"] = lambda x: eval_str_list(x, int) - elif "float" in str(inter_type): - kwargs["type"] = lambda x: eval_str_list(x, float) - elif "str" in str(inter_type): - kwargs["type"] = lambda x: eval_str_list(x, str) - else: - raise NotImplementedError( - "parsing of type " + str(inter_type) + " is not implemented" - ) - if field_default is not MISSING: - kwargs["default"] = ( - ",".join(map(str, field_default)) - if field_default is not None - else None - ) - elif ( - isinstance(inter_type, type) and issubclass(inter_type, Enum) - ) or "Enum" in str(inter_type): - kwargs["type"] = str - if field_default is not MISSING: - if isinstance(field_default, Enum): - kwargs["default"] = field_default.value - else: - kwargs["default"] = field_default - elif inter_type is bool: - kwargs["action"] = ( - "store_false" if field_default is True else "store_true" - ) - kwargs["default"] = field_default - else: - kwargs["type"] = inter_type - if field_default is not MISSING: - kwargs["default"] = field_default - - # build the help with the hierarchical prefix - if with_prefix is not None and with_prefix != "" and field_help is not None: - field_help = with_prefix[2:] + ": " + field_help - - kwargs["help"] = field_help - if field_const is not None: - kwargs["const"] = field_const - kwargs["nargs"] = "?" - - return kwargs - - for k in dataclass_instance._get_all_attributes(): - field_name = argparse_name(dataclass_instance._get_name(k)) - field_type = dataclass_instance._get_type(k) - if field_name is None: - continue - elif inspect.isclass(field_type) and issubclass(field_type, FairseqDataclass): - # for fields that are of type FairseqDataclass, we can recursively - # add their fields to the namespace (so we add the args from model, task, etc. to the root namespace) - prefix = None - if with_prefix is not None: - # if a prefix is specified, then we don't want to copy the subfields directly to the root namespace - # but we prefix them with the name of the current field. - prefix = field_name - gen_parser_from_dataclass(parser, field_type(), delete_default, prefix) - continue - - kwargs = get_kwargs_from_dc(dataclass_instance, k) - - field_args = [field_name] - alias = dataclass_instance._get_argparse_alias(k) - if alias is not None: - field_args.append(alias) - - if "default" in kwargs: - if isinstance(kwargs["default"], str) and kwargs["default"].startswith( - "${" - ): - if kwargs["help"] is None: - # this is a field with a name that will be added elsewhere - continue - else: - del kwargs["default"] - if delete_default and "default" in kwargs: - del kwargs["default"] - try: - parser.add_argument(*field_args, **kwargs) - except ArgumentError: - pass - - -def _set_legacy_defaults(args, cls): - """Helper to set default arguments based on *add_args*.""" - if not hasattr(cls, "add_args"): - return - - import argparse - - parser = argparse.ArgumentParser( - argument_default=argparse.SUPPRESS, allow_abbrev=False - ) - cls.add_args(parser) - # copied from argparse.py: - defaults = argparse.Namespace() - for action in parser._actions: - if action.dest is not argparse.SUPPRESS: - if not hasattr(defaults, action.dest): - if action.default is not argparse.SUPPRESS: - setattr(defaults, action.dest, action.default) - for key, default_value in vars(defaults).items(): - if not hasattr(args, key): - setattr(args, key, default_value) - - -def _override_attr( - sub_node: str, data_class: Type[FairseqDataclass], args: Namespace -) -> List[str]: - overrides = [] - - if not inspect.isclass(data_class) or not issubclass(data_class, FairseqDataclass): - return overrides - - def get_default(f): - if not isinstance(f.default_factory, _MISSING_TYPE): - return f.default_factory() - return f.default - - for k, v in data_class.__dataclass_fields__.items(): - if k.startswith("_"): - # private member, skip - continue - - val = get_default(v) if not hasattr(args, k) else getattr(args, k) - - field_type = interpret_dc_type(v.type) - if ( - isinstance(val, str) - and not val.startswith("${") # not interpolation - and field_type != str - and ( - not inspect.isclass(field_type) or not issubclass(field_type, Enum) - ) # not choices enum - ): - # upgrade old models that stored complex parameters as string - val = ast.literal_eval(val) - - if isinstance(val, tuple): - val = list(val) - - v_type = getattr(v.type, "__origin__", None) - if ( - (v_type is List or v_type is list or v_type is Optional) - # skip interpolation - and not (isinstance(val, str) and val.startswith("${")) - ): - # if type is int but val is float, then we will crash later - try to convert here - if hasattr(v.type, "__args__"): - t_args = v.type.__args__ - if len(t_args) == 1 and (t_args[0] is float or t_args[0] is int): - val = list(map(t_args[0], val)) - elif val is not None and ( - field_type is int or field_type is bool or field_type is float - ): - try: - val = field_type(val) - except: - pass # ignore errors here, they are often from interpolation args - - if val is None: - overrides.append("{}.{}=null".format(sub_node, k)) - elif val == "": - overrides.append("{}.{}=''".format(sub_node, k)) - elif isinstance(val, str): - val = val.replace("'", r"\'") - overrides.append("{}.{}='{}'".format(sub_node, k, val)) - elif isinstance(val, FairseqDataclass): - overrides += _override_attr(f"{sub_node}.{k}", type(val), args) - elif isinstance(val, Namespace): - sub_overrides, _ = override_module_args(val) - for so in sub_overrides: - overrides.append(f"{sub_node}.{k}.{so}") - else: - overrides.append("{}.{}={}".format(sub_node, k, val)) - - return overrides - - -def migrate_registry( - name, value, registry, args, overrides, deletes, use_name_as_val=False -): - if value in registry: - overrides.append("{}={}".format(name, value)) - overrides.append("{}._name={}".format(name, value)) - overrides.extend(_override_attr(name, registry[value], args)) - elif use_name_as_val and value is not None: - overrides.append("{}={}".format(name, value)) - else: - deletes.append(name) - - -def override_module_args(args: Namespace) -> Tuple[List[str], List[str]]: - """use the field in args to overrides those in cfg""" - overrides = [] - deletes = [] - - for k in FairseqConfig.__dataclass_fields__.keys(): - overrides.extend( - _override_attr(k, FairseqConfig.__dataclass_fields__[k].type, args) - ) - - if args is not None: - if hasattr(args, "task"): - from fairseq.tasks import TASK_DATACLASS_REGISTRY - - migrate_registry( - "task", args.task, TASK_DATACLASS_REGISTRY, args, overrides, deletes - ) - else: - deletes.append("task") - - # these options will be set to "None" if they have not yet been migrated - # so we can populate them with the entire flat args - CORE_REGISTRIES = {"criterion", "optimizer", "lr_scheduler"} - - from fairseq.registry import REGISTRIES - - for k, v in REGISTRIES.items(): - if hasattr(args, k): - migrate_registry( - k, - getattr(args, k), - v["dataclass_registry"], - args, - overrides, - deletes, - use_name_as_val=k not in CORE_REGISTRIES, - ) - else: - deletes.append(k) - - no_dc = True - if hasattr(args, "arch"): - from fairseq.models import ARCH_MODEL_NAME_REGISTRY, ARCH_MODEL_REGISTRY - - if args.arch in ARCH_MODEL_REGISTRY: - m_cls = ARCH_MODEL_REGISTRY[args.arch] - dc = getattr(m_cls, "__dataclass", None) - if dc is not None: - m_name = ARCH_MODEL_NAME_REGISTRY[args.arch] - overrides.append("model={}".format(m_name)) - overrides.append("model._name={}".format(args.arch)) - # override model params with those exist in args - overrides.extend(_override_attr("model", dc, args)) - no_dc = False - if no_dc: - deletes.append("model") - - return overrides, deletes - - -class omegaconf_no_object_check: - def __init__(self): - self.old_is_primitive = _utils.is_primitive_type - - def __enter__(self): - _utils.is_primitive_type = lambda _: True - - def __exit__(self, type, value, traceback): - _utils.is_primitive_type = self.old_is_primitive - - -def convert_namespace_to_omegaconf(args: Namespace) -> DictConfig: - """Convert a flat argparse.Namespace to a structured DictConfig.""" - - # Here we are using field values provided in args to override counterparts inside config object - overrides, deletes = override_module_args(args) - - # configs will be in fairseq/config after installation - config_path = os.path.join("..", "config") - - GlobalHydra.instance().clear() - - with initialize(config_path=config_path): - try: - composed_cfg = compose("config", overrides=overrides, strict=False) - except: - logger.error("Error when composing. Overrides: " + str(overrides)) - raise - - for k in deletes: - composed_cfg[k] = None - - cfg = OmegaConf.create( - OmegaConf.to_container(composed_cfg, resolve=True, enum_to_str=True) - ) - - # hack to be able to set Namespace in dict config. this should be removed when we update to newer - # omegaconf version that supports object flags, or when we migrate all existing models - from omegaconf import _utils - - with omegaconf_no_object_check(): - if cfg.task is None and getattr(args, "task", None): - cfg.task = Namespace(**vars(args)) - from fairseq.tasks import TASK_REGISTRY - - _set_legacy_defaults(cfg.task, TASK_REGISTRY[args.task]) - cfg.task._name = args.task - if cfg.model is None and getattr(args, "arch", None): - cfg.model = Namespace(**vars(args)) - from fairseq.models import ARCH_MODEL_REGISTRY - - _set_legacy_defaults(cfg.model, ARCH_MODEL_REGISTRY[args.arch]) - cfg.model._name = args.arch - if cfg.optimizer is None and getattr(args, "optimizer", None): - cfg.optimizer = Namespace(**vars(args)) - from fairseq.optim import OPTIMIZER_REGISTRY - - _set_legacy_defaults(cfg.optimizer, OPTIMIZER_REGISTRY[args.optimizer]) - cfg.optimizer._name = args.optimizer - if cfg.lr_scheduler is None and getattr(args, "lr_scheduler", None): - cfg.lr_scheduler = Namespace(**vars(args)) - from fairseq.optim.lr_scheduler import LR_SCHEDULER_REGISTRY - - _set_legacy_defaults( - cfg.lr_scheduler, LR_SCHEDULER_REGISTRY[args.lr_scheduler] - ) - cfg.lr_scheduler._name = args.lr_scheduler - if cfg.criterion is None and getattr(args, "criterion", None): - cfg.criterion = Namespace(**vars(args)) - from fairseq.criterions import CRITERION_REGISTRY - - _set_legacy_defaults(cfg.criterion, CRITERION_REGISTRY[args.criterion]) - cfg.criterion._name = args.criterion - - OmegaConf.set_struct(cfg, True) - return cfg - - -def overwrite_args_by_name(cfg: DictConfig, overrides: Dict[str, any]): - # this will be deprecated when we get rid of argparse and model_overrides logic - - from fairseq.registry import REGISTRIES - - with open_dict(cfg): - for k in cfg.keys(): - # "k in cfg" will return false if its a "mandatory value (e.g. ???)" - if k in cfg and isinstance(cfg[k], DictConfig): - if k in overrides and isinstance(overrides[k], dict): - for ok, ov in overrides[k].items(): - if isinstance(ov, dict) and cfg[k][ok] is not None: - overwrite_args_by_name(cfg[k][ok], ov) - else: - cfg[k][ok] = ov - else: - overwrite_args_by_name(cfg[k], overrides) - elif k in cfg and isinstance(cfg[k], Namespace): - for override_key, val in overrides.items(): - setattr(cfg[k], override_key, val) - elif k in overrides: - if ( - k in REGISTRIES - and overrides[k] in REGISTRIES[k]["dataclass_registry"] - ): - cfg[k] = DictConfig( - REGISTRIES[k]["dataclass_registry"][overrides[k]] - ) - overwrite_args_by_name(cfg[k], overrides) - cfg[k]._name = overrides[k] - else: - cfg[k] = overrides[k] - - -def merge_with_parent(dc: FairseqDataclass, cfg: DictConfig, remove_missing=False): - if remove_missing: - - if is_dataclass(dc): - target_keys = set(dc.__dataclass_fields__.keys()) - else: - target_keys = set(dc.keys()) - - with open_dict(cfg): - for k in list(cfg.keys()): - if k not in target_keys: - del cfg[k] - - merged_cfg = OmegaConf.merge(dc, cfg) - merged_cfg.__dict__["_parent"] = cfg.__dict__["_parent"] - OmegaConf.set_struct(merged_cfg, True) - return merged_cfg diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/distributed/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/distributed/__init__.py deleted file mode 100644 index eb34a9b3de7e2c442e3e0e835a89699d93ef6841..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/distributed/__init__.py +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from .distributed_timeout_wrapper import DistributedTimeoutWrapper -from .fully_sharded_data_parallel import ( - FullyShardedDataParallel, - fsdp_enable_wrap, - fsdp_wrap, -) -from .legacy_distributed_data_parallel import LegacyDistributedDataParallel -from .module_proxy_wrapper import ModuleProxyWrapper -from .tpu_distributed_data_parallel import TPUDistributedDataParallel - -__all__ = [ - "DistributedTimeoutWrapper", - "fsdp_enable_wrap", - "fsdp_wrap", - "FullyShardedDataParallel", - "LegacyDistributedDataParallel", - "ModuleProxyWrapper", - "TPUDistributedDataParallel", -] diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/distributed/distributed_timeout_wrapper.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/distributed/distributed_timeout_wrapper.py deleted file mode 100644 index f02c260389e867675d4c76749f849a30b2159df0..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/distributed/distributed_timeout_wrapper.py +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import logging -import os -import signal -import threading - -from torch import nn - -logger = logging.getLogger(__name__) - - -class DistributedTimeoutWrapper(nn.Module): - """ - A wrapper that kills the process if no progress is made within a given - *timeout*. The timer is reset every time :func:`forward` is called. - - Usage:: - - module = DistributedTimeoutWrapper(module, timeout=30) - x = module(input) - time.sleep(20) # safe - x = module(input) - time.sleep(45) # job will be killed before this returns - - Args: - module (nn.Module): module to wrap - timeout (int): number of seconds before killing the process - (set to a value <= 0 to disable the timeout) - signal (Optional): signal to send once timeout is triggered - """ - - def __init__(self, module: nn.Module, timeout: int, signal=signal.SIGINT): - super().__init__() - self.module = module - self.timeout = timeout - self.signal = signal - - if timeout > 0: - self._heartbeat = threading.Event() - self._heartbeat_thread = threading.Thread( - target=self._check_heartbeat, - args=(os.getpid(),), - daemon=True, - ) - self._heartbeat_thread.start() - self._terminated = False - else: - self._heartbeat = None - self._heartbeat_thread = None - - def __del__(self): - self.stop_timeout() - - def __getattr__(self, name): - """Forward missing attributes to wrapped module.""" - try: - return super().__getattr__(name) # defer to nn.Module's logic - except AttributeError: - return getattr(self.module, name) - - def stop_timeout(self): - if self._heartbeat_thread is not None: - self._terminated = True - self._heartbeat_thread.join() - - def state_dict(self, *args, **kwargs): - return self.module.state_dict(*args, **kwargs) - - def load_state_dict(self, *args, **kwargs): - return self.module.load_state_dict(*args, **kwargs) - - def forward(self, *args, **kwargs): - if self._heartbeat is not None: - self._heartbeat.set() - return self.module(*args, **kwargs) - - def _check_heartbeat(self, parent_pid): - self._heartbeat.wait() # wait for the first forward pass - while True: - self._heartbeat.clear() - success = self._heartbeat.wait(timeout=self.timeout) - if self._terminated: - break - elif not success: - logger.error( - ( - "Killing job for not making progress in {} seconds. " - "Set --heartbeat-timeout=-1 to disable this timeout." - ).format(int(self.timeout)) - ) - os.kill(parent_pid, self.signal) - return diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/distributed/fully_sharded_data_parallel.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/distributed/fully_sharded_data_parallel.py deleted file mode 100644 index fac2108633d3fd15a2364caa32dda26b0fdd17f1..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/distributed/fully_sharded_data_parallel.py +++ /dev/null @@ -1,134 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import contextlib -from typing import Optional - -import torch -from fairseq.dataclass.configs import DistributedTrainingConfig -from fairseq.distributed import utils as dist_utils - -try: - from fairscale.nn.data_parallel import FullyShardedDataParallel as FSDP - - has_FSDP = True -except ImportError: - FSDP = torch.nn.Module - has_FSDP = False - - -class FullyShardedDataParallel(FSDP): - """ - A small wrapper around fairscale's FullyShardedDataParallel (FSDP) with some - fairseq-specific checkpoint saving/loading logic. - - Args: - use_sharded_state (bool): if True, then ``state_dict`` will return - ``FSDP.local_state_dict`` and ``load_state_dict`` will call - ``FSDP.load_local_state_dict``. Otherwise, ``state_dict`` will - return the full model weights on data parallel rank 0 (empty on - other ranks) and ``load_state_dict`` will broadcast model weights - from rank 0 to other ranks. - """ - - def __init__(self, *args, use_sharded_state: bool = False, **kwargs): - if not has_FSDP: - raise ImportError( - "Cannot find FullyShardedDataParallel. " - "Please install fairscale with: pip install fairscale" - ) - super().__init__(*args, **kwargs) - self.use_sharded_state = use_sharded_state - - @property - def unwrapped_module(self) -> torch.nn.Module: - if self.flatten_parameters: - return self.module.module - else: - return self.module - - def state_dict(self, destination=None, prefix="", keep_vars=False): - if self.use_sharded_state: - return super().local_state_dict( - destination=destination, prefix=prefix, keep_vars=keep_vars - ) - else: - if self.rank == 0: - return super().state_dict( - destination=destination, prefix=prefix, keep_vars=keep_vars - ) - else: - # We must call state_dict() due to use of communication - # primitives. But we don't use the result. - super().state_dict() - return destination or {} - - def load_state_dict(self, state_dict, strict=True, model_cfg=None): - if self.use_sharded_state: - return super().load_local_state_dict(state_dict, strict=strict) - else: - state_dict = dist_utils.broadcast_object( - state_dict, src_rank=0, group=self.process_group - ) - return super().load_state_dict(state_dict, strict=strict) - - -@contextlib.contextmanager -def fsdp_enable_wrap(cfg: DistributedTrainingConfig): - try: - from fairscale.nn import enable_wrap - except ImportError: - raise ImportError( - "Cannot find FullyShardedDataParallel. " - "Please install fairscale with: pip install fairscale" - ) - if cfg.memory_efficient_fp16: - assert cfg.fp16 # memory_efficient_fp16 should imply fp16 - group = dist_utils.get_data_parallel_group() - if group is None and cfg.distributed_world_size == 1: - from fairscale.utils.testing import DummyProcessGroup - - group = DummyProcessGroup(rank=0, size=1) - fsdp_config = { - "process_group": group, - "reshard_after_forward": not cfg.no_reshard_after_forward, - "mixed_precision": cfg.fp16 and not cfg.memory_efficient_fp16, - "fp32_reduce_scatter": cfg.fp32_reduce_scatter, - "flatten_parameters": not cfg.not_fsdp_flatten_parameters, - "cpu_offload": cfg.cpu_offload, - "compute_dtype": torch.float16 if cfg.fp16 else torch.float32, - "bucket_cap_mb": cfg.bucket_cap_mb, - "state_dict_device": torch.device("cpu"), # reduce GPU mem usage - } - with enable_wrap( - wrapper_cls=FullyShardedDataParallel, - use_sharded_state=cfg.use_sharded_state, - **fsdp_config, - ): - yield - - -def fsdp_wrap(module, min_num_params: Optional[int] = None, **kwargs): - """ - Helper to wrap layers/modules in FSDP. This falls back to a no-op if - fairscale is not available. - - Args: - module (nn.Module): module to (maybe) wrap - min_num_params (int, Optional): minimum number of layer params to wrap - """ - try: - from fairscale.nn import wrap - - if min_num_params is not None: - num_params = sum(p.numel() for p in module.parameters()) - if num_params >= min_num_params: - return wrap(module, **kwargs) - else: - return module - else: - return wrap(module, **kwargs) - except ImportError: - return module diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/distributed/legacy_distributed_data_parallel.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/distributed/legacy_distributed_data_parallel.py deleted file mode 100644 index 3bdb00e9d8aa94478b9f92f62e5c26ac6f3349eb..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/distributed/legacy_distributed_data_parallel.py +++ /dev/null @@ -1,164 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -""" -A modified version of the legacy DistributedDataParallel module that uses c10d -communication primitives. This version is simpler than the latest PyTorch -version and is useful for debugging. Notably it does not overlap gradient -communication with the backward pass, which makes it slower but more robust -than the PyTorch version. - -This version also supports the *no_sync* context manager, which allows faster -training with `--update-freq`. -""" - -from collections import OrderedDict -from contextlib import contextmanager - -import torch -from fairseq.distributed import utils -from torch import nn - - -class LegacyDistributedDataParallel(nn.Module): - """Implements distributed data parallelism at the module level. - - A simplified version of :class:`torch.nn.parallel.DistributedDataParallel`. - This version uses a c10d process group for communication and does not - broadcast buffers. - - Args: - module (~torch.nn.Module): module to be parallelized - process_group: the c10d process group to be used for distributed data - parallel all-reduction. - buffer_size (int, optional): number of elements to buffer before - performing all-reduce (default: 256M). - """ - - def __init__(self, module, process_group, buffer_size=2**28): - super().__init__() - - self.module = module - self.process_group = process_group - self.world_size = utils.get_world_size(self.process_group) - - # Never use a bigger buffer than the number of model params - self.buffer_size = min(buffer_size, sum(p.numel() for p in module.parameters())) - self.buffer = None - - # We can also forcibly accumulate grads locally and only do the - # all-reduce at some later time - self.accumulate_grads = False - - # make per-device lists of parameters - paramlists = OrderedDict() - for param in self.module.parameters(): - device = param.device - if paramlists.get(device) is None: - paramlists[device] = [] - paramlists[device] += [param] - self.per_device_params = list(paramlists.values()) - - @contextmanager - def no_sync(self): - """A context manager to disable gradient synchronization.""" - old_accumulate_grads = self.accumulate_grads - self.accumulate_grads = True - yield - self.accumulate_grads = old_accumulate_grads - - def forward(self, *inputs, **kwargs): - return self.module(*inputs, **kwargs) - - def all_reduce_grads(self): - """ - This function must be called explicitly after backward to reduce - gradients. There is no automatic hook like c10d. - """ - - def all_reduce_params(params): - buffer = self.buffer - nonzero_buffer = False - if len(params) > 1: - offset = 0 - for p in params: - sz = p.numel() - if p.grad is not None: - buffer[offset : offset + sz].copy_(p.grad.data.view(-1)) - nonzero_buffer = True - else: - buffer[offset : offset + sz].zero_() - offset += sz - else: - # we only have a single grad to all-reduce - p = params[0] - if p.grad is not None: - buffer = p.grad.data - nonzero_buffer = True - elif p.numel() <= self.buffer.numel(): - buffer = buffer[: p.numel()] - buffer.zero_() - else: - buffer = torch.zeros_like(p) - - if nonzero_buffer: - buffer.div_(self.world_size) - - utils.all_reduce(buffer, self.process_group) - - # copy all-reduced grads back into their original place - offset = 0 - for p in params: - sz = p.numel() - if p.grad is not None: - p.grad.data.copy_(buffer[offset : offset + sz].view_as(p)) - else: - p.grad = buffer[offset : offset + sz].view_as(p).clone() - offset += sz - - def reduction_fn(): - # This function only needs to be called once - if self.accumulate_grads: - return - - if self.buffer is None: - self.buffer = next(self.module.parameters()).new(self.buffer_size) - - for params in self.per_device_params: - # All-reduce the gradients in buckets - offset = 0 - buffered_params = [] - for param in params: - if not param.requires_grad: - continue - if param.grad is None: - param.grad = torch.zeros_like(param) - - if hasattr(param, "expert"): - # Skip gradient sync for unshared parameters - continue - - if param.grad.requires_grad: - raise RuntimeError( - "DistributedDataParallel only works " - "with gradients that don't require " - "grad" - ) - sz = param.numel() - if sz > self.buffer.numel(): - # all-reduce big params directly - all_reduce_params([param]) - else: - if offset + sz > self.buffer.numel(): - all_reduce_params(buffered_params) - offset = 0 - buffered_params.clear() - buffered_params.append(param) - offset += sz - - if len(buffered_params) > 0: - all_reduce_params(buffered_params) - - reduction_fn() diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/distributed/module_proxy_wrapper.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/distributed/module_proxy_wrapper.py deleted file mode 100644 index 904dc0c202e09db244518836c0f061e0850cad61..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/distributed/module_proxy_wrapper.py +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from torch import nn - - -class ModuleProxyWrapper(nn.Module): - """ - Wrap a DistributedDataParallel module and forward requests for missing - attributes to the module wrapped by DDP (the twice-wrapped module). - Also forward calls to :func:`state_dict` and :func:`load_state_dict`. - - Usage:: - - module.xyz = "hello world" - wrapped_module = DistributedDataParallel(module, **ddp_args) - wrapped_module = ModuleProxyWrapper(wrapped_module) - assert wrapped_module.xyz == "hello world" - assert wrapped_module.state_dict().keys() == module.state_dict().keys() - - Args: - module (nn.Module): module to wrap - """ - - def __init__(self, module: nn.Module): - super().__init__() - assert hasattr( - module, "module" - ), "ModuleProxyWrapper expects input to wrap another module" - self.module = module - - def __getattr__(self, name): - """Forward missing attributes to twice-wrapped module.""" - try: - # defer to nn.Module's logic - return super().__getattr__(name) - except AttributeError: - try: - # forward to the once-wrapped module - return getattr(self.module, name) - except AttributeError: - # forward to the twice-wrapped module - return getattr(self.module.module, name) - - def state_dict(self, *args, **kwargs): - """Forward to the twice-wrapped module.""" - return self.module.module.state_dict(*args, **kwargs) - - def load_state_dict(self, *args, **kwargs): - """Forward to the twice-wrapped module.""" - return self.module.module.load_state_dict(*args, **kwargs) - - def forward(self, *args, **kwargs): - return self.module(*args, **kwargs) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/distributed/tpu_distributed_data_parallel.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/distributed/tpu_distributed_data_parallel.py deleted file mode 100644 index 67fff420127fd0029b221fdb6eb3c52bc532eb6d..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/distributed/tpu_distributed_data_parallel.py +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import torch -from fairseq.distributed import utils -from torch import nn - - -class TPUDistributedDataParallel(nn.Module): - def __init__(self, module, process_group): - super().__init__() - self.module = module - self.process_group = process_group - self.world_size = utils.get_world_size(self.process_group) - - def forward(self, *inputs, **kwargs): - return self.module(*inputs, **kwargs) - - def all_reduce_grads(self): - gradients = [] - for p in self.parameters(): - if not p.requires_grad: - continue - if p.grad is None: - p.grad = torch.zeros_like(p) - if p.grad.requires_grad: - raise RuntimeError( - "TPUDistributedDataParallel only works with gradients that don't " - "require grad" - ) - gradients.append(p.grad) - - import torch_xla.core.xla_model as xm - - xm.all_reduce( - "sum", - gradients, - scale=1.0 / self.world_size, - groups=self.process_group[1], - ) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/distributed/utils.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/distributed/utils.py deleted file mode 100644 index 2c52f76aa7d60250101cdaee9a80ed3c9d58a4fa..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/distributed/utils.py +++ /dev/null @@ -1,808 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import io -import logging -import os -import pickle -import random -import socket -import struct -import subprocess -import warnings -from argparse import Namespace -from collections import OrderedDict -from dataclasses import dataclass -from typing import Any, Dict, List, Mapping, Optional - -import torch -import torch.distributed as dist -from fairseq.dataclass.configs import DistributedTrainingConfig, FairseqConfig -from omegaconf import open_dict - -try: - import torch_xla.core.xla_model as xm -except ImportError: - xm = None - - -# Flag to indicate if we're using Megatron -# NOTE: this is a temporary hack until we move away from Megatron's model parallel init -_USE_MEGATRON = False - -# Whether to use XLA ops (e.g., on TPUs) instead of CUDA ops. -_USE_XLA = False - - -logger = logging.getLogger(__name__) - - -def is_master(cfg: DistributedTrainingConfig): - return cfg.distributed_rank == 0 - - -def infer_init_method(cfg: DistributedTrainingConfig, force_distributed=False): - if cfg.distributed_init_method is not None or cfg.tpu: - return - - num_pipelines_per_node = None - if cfg.pipeline_model_parallel: - num_pipeline_devices, num_pipelines_per_node = _pipeline_parallel_pre_init(cfg) - - if all( - key in os.environ - for key in ["MASTER_ADDR", "MASTER_PORT", "WORLD_SIZE", "RANK"] - ): - # support torch.distributed.launch - _infer_torch_distributed_launch_init(cfg) - elif cfg.distributed_port > 0: - # we can determine the init method automatically for Slurm - _infer_slurm_init(cfg, num_pipelines_per_node) - elif cfg.distributed_world_size > 1 or force_distributed: - # fallback for single node with multiple GPUs - _infer_single_node_init(cfg) - - if cfg.pipeline_model_parallel: - _pipeline_parallel_post_init(cfg, num_pipeline_devices, num_pipelines_per_node) - elif not cfg.distributed_no_spawn: - with open_dict(cfg): - cfg.distributed_num_procs = min( - torch.cuda.device_count(), cfg.distributed_world_size - ) - - -def _infer_torch_distributed_launch_init(cfg: DistributedTrainingConfig): - cfg.distributed_init_method = "env://" - cfg.distributed_world_size = int(os.environ["WORLD_SIZE"]) - cfg.distributed_rank = int(os.environ["RANK"]) - # processes are created by torch.distributed.launch - cfg.distributed_no_spawn = True - - -def _infer_slurm_init(cfg: DistributedTrainingConfig, num_pipelines_per_node): - node_list = os.environ.get("SLURM_STEP_NODELIST") - if node_list is None: - node_list = os.environ.get("SLURM_JOB_NODELIST") - if node_list is not None: - try: - hostnames = subprocess.check_output( - ["scontrol", "show", "hostnames", node_list] - ) - cfg.distributed_init_method = "tcp://{host}:{port}".format( - host=hostnames.split()[0].decode("utf-8"), - port=cfg.distributed_port, - ) - nnodes = int(os.environ.get("SLURM_NNODES")) - ntasks_per_node = os.environ.get("SLURM_NTASKS_PER_NODE") - if ntasks_per_node is not None: - ntasks_per_node = int(ntasks_per_node) - else: - ntasks = int(os.environ.get("SLURM_NTASKS")) - nnodes = int(os.environ.get("SLURM_NNODES")) - assert ntasks % nnodes == 0 - ntasks_per_node = int(ntasks / nnodes) - if ntasks_per_node == 1: - gpus_per_node = torch.cuda.device_count() - node_id = int(os.environ.get("SLURM_NODEID")) - cfg.distributed_rank = node_id * gpus_per_node - cfg.distributed_world_size = nnodes * gpus_per_node - elif cfg.pipeline_model_parallel: - assert ntasks_per_node == num_pipelines_per_node, ( - "SLURM --ntasks-per-node must match number of pipelines per " - "node (={})".format(num_pipelines_per_node) - ) - cfg.distributed_no_spawn = True - # For 4-way MP on nodes with 8 GPUs, ranks will be [0, 1] on - # the first node, [1, 2] on the second node, etc. This - # matches torch.distributed.launch. - node_id = int(os.environ.get("SLURM_NODEID")) - local_id = int(os.environ.get("SLURM_LOCALID")) - cfg.distributed_rank = node_id * num_pipelines_per_node + local_id - # In the above example, device_id will always be in [0, 1], - # which also matches torch.distributed.launch. - cfg.device_id = local_id - # We also want to set distributed_world_size to be the total - # number of pipelines across all nodes. - cfg.distributed_world_size = nnodes * num_pipelines_per_node - else: - assert ntasks_per_node == cfg.distributed_world_size // nnodes - cfg.distributed_no_spawn = True - cfg.distributed_rank = int(os.environ.get("SLURM_PROCID")) - cfg.device_id = int(os.environ.get("SLURM_LOCALID")) - except subprocess.CalledProcessError as e: # scontrol failed - raise e - except FileNotFoundError: # Slurm is not installed - pass - - -def _infer_single_node_init(cfg: DistributedTrainingConfig): - assert ( - cfg.distributed_world_size <= torch.cuda.device_count() - ), f"world size is {cfg.distributed_world_size} but have {torch.cuda.device_count()} available devices" - port = random.randint(10000, 20000) - cfg.distributed_init_method = "tcp://localhost:{port}".format(port=port) - - -def _pipeline_parallel_pre_init(cfg: DistributedTrainingConfig): - from fairseq import utils - - balance_exists = ( - cfg.pipeline_balance is not None - or cfg.pipeline_encoder_balance is not None - or cfg.pipeline_decoder_balance is not None - ) - devices_exist = ( - cfg.pipeline_devices is not None - or cfg.pipeline_encoder_devices is not None - or cfg.pipeline_decoder_devices is not None - ) - if not balance_exists: - raise ValueError( - "--pipeline-balance is currently required for pipeline model parallelism" - ) - if not devices_exist: - raise ValueError( - "--pipeline-devices is currently required for pipeline model parallelism" - ) - - cfg.pipeline_balance = utils.eval_str_list(cfg.pipeline_balance, type=int) - if cfg.pipeline_devices is not None: - cfg.pipeline_devices = utils.eval_str_list(cfg.pipeline_devices, type=int) - num_pipeline_devices = len(set(cfg.pipeline_devices)) - else: - cfg.pipeline_encoder_devices = utils.eval_str_list( - cfg.pipeline_encoder_devices, type=int - ) - cfg.pipeline_decoder_devices = utils.eval_str_list( - cfg.pipeline_decoder_devices, type=int - ) - num_pipeline_devices = len( - set(cfg.pipeline_encoder_devices + cfg.pipeline_decoder_devices) - ) - gpus_per_node = torch.cuda.device_count() - assert ( - gpus_per_node >= num_pipeline_devices - and gpus_per_node % num_pipeline_devices == 0 - ), ( - "the number of unique device IDs in --pipeline-devices must evenly divide " - "the number of GPUs per node (multi-node pipelining is not yet supported)" - ) - num_pipelines_per_node = gpus_per_node // num_pipeline_devices - return num_pipeline_devices, num_pipelines_per_node - - -def _pipeline_parallel_post_init( - cfg: DistributedTrainingConfig, num_pipeline_devices, num_pipelines_per_node -): - if not cfg.distributed_no_spawn: - # When distributed_no_spawn is False, we expect distributed_rank and - # distributed_world_size to be based on the total number of GPUs, so - # we need to correct them to be based on the number of pipelines. - assert cfg.distributed_world_size % num_pipeline_devices == 0 - cfg.distributed_world_size = cfg.distributed_world_size // num_pipeline_devices - # In the case of 4-way MP on nodes with 8 GPUs, we want - # distributed_rank to be the starting GPU index for each pipeline - # i.e., 0, 2, ... - gpus_per_node = torch.cuda.device_count() - assert cfg.distributed_rank % gpus_per_node == 0 - assert cfg.distributed_rank % num_pipeline_devices == 0 - - with open_dict(cfg): - cfg.distributed_rank = cfg.distributed_rank // num_pipeline_devices - # launch one process per pipeline - cfg.distributed_num_procs = num_pipelines_per_node - - # if we have 4-way MP on a node with 8 GPUs, we want device_ids to be 0 - # and 4, indicating the starting device IDs for each pipeline - cfg.device_id *= num_pipeline_devices - - if cfg.device_id > 0: - # if there's multiple pipelines on a node (e.g., 4-way MP on an 8 - # GPU node), we need to adjust pipeline_devices accordingly - logger.debug( - "setting CUDA device={} on rank {}".format( - cfg.device_id, cfg.distributed_rank - ) - ) - torch.cuda.set_device(cfg.device_id) - with open_dict(cfg): - cfg.pipeline_devices = [cfg.device_id + d for d in cfg.pipeline_devices] - logger.info( - "setting pipeline_devices={} on rank {}".format( - cfg.pipeline_devices, cfg.distributed_rank - ) - ) - - -def distributed_init(cfg: FairseqConfig): - if isinstance(cfg, Namespace): - from fairseq.dataclass.utils import convert_namespace_to_omegaconf - - cfg = convert_namespace_to_omegaconf(cfg) - - if not cfg.common.tpu: - if torch.distributed.is_available() and torch.distributed.is_initialized(): - warnings.warn( - "Distributed is already initialized, cannot initialize twice!" - ) - else: - logger.info( - "distributed init (rank {}): {}".format( - cfg.distributed_training.distributed_rank, - cfg.distributed_training.distributed_init_method, - ) - ) - dist.init_process_group( - backend=cfg.distributed_training.distributed_backend, - init_method=cfg.distributed_training.distributed_init_method, - world_size=cfg.distributed_training.distributed_world_size, - rank=cfg.distributed_training.distributed_rank, - ) - logger.info( - "initialized host {} as rank {}".format( - socket.gethostname(), - cfg.distributed_training.distributed_rank, - ) - ) - - # perform a dummy all-reduce to initialize the NCCL communicator - if torch.cuda.is_available(): - dist.all_reduce(torch.zeros(1).cuda()) - - cfg.distributed_training.distributed_rank = torch.distributed.get_rank() - else: - assert xm.xrt_world_size() == cfg.distributed_training.distributed_world_size - global _USE_XLA - _USE_XLA = True - cfg.distributed_training.device_id = xm.get_local_ordinal() - cfg.distributed_training.distributed_rank = xm.get_ordinal() - xm.rendezvous("distributed_init") # wait for all workers - - if is_master(cfg.distributed_training): - logging.getLogger().setLevel(logging.INFO) - else: - logging.getLogger().setLevel(logging.WARNING) - - if cfg.common.model_parallel_size > 1: - try: - from fairseq.model_parallel.megatron.mpu import ( - initialize_model_parallel, - model_parallel_cuda_manual_seed, - ) - except ImportError: - raise ImportError( - "\n\nPlease install the megatron submodule:" - "\n\n git submodule update --init " - "fairseq/model_parallel/megatron" - ) - global _USE_MEGATRON - _USE_MEGATRON = True - initialize_model_parallel(cfg.common.model_parallel_size) - model_parallel_cuda_manual_seed(cfg.common.seed) - model_part_number = get_model_parallel_rank() - cfg.checkpoint.checkpoint_suffix += "-model_part-{0}".format(model_part_number) - - if hasattr(cfg, "model") and getattr(cfg.model, "base_layers", 0) > 0: - cfg.checkpoint.checkpoint_suffix = ( - f"-rank-{cfg.distributed_training.distributed_rank}" - ) - - return cfg.distributed_training.distributed_rank - - -def distributed_main(i, main, cfg: FairseqConfig, kwargs): - cfg.distributed_training.device_id = i - if torch.cuda.is_available() and not cfg.common.cpu and not cfg.common.tpu: - torch.cuda.set_device(cfg.distributed_training.device_id) - if cfg.distributed_training.distributed_rank is None: # torch.multiprocessing.spawn - cfg.distributed_training.distributed_rank = kwargs.pop("start_rank", 0) + i - - cfg.distributed_training.distributed_rank = distributed_init(cfg) - - after_distributed_init_fn = kwargs.pop("after_distributed_init_fn", None) - if after_distributed_init_fn: - cfg = after_distributed_init_fn(cfg) - - main(cfg, **kwargs) - - if torch.distributed.is_initialized(): - torch.distributed.barrier(get_global_group()) - - -def call_main(cfg: FairseqConfig, main, **kwargs): - if cfg.distributed_training.distributed_init_method is None: - infer_init_method(cfg.distributed_training) - - if cfg.distributed_training.distributed_init_method is not None: - # distributed training - if not cfg.distributed_training.distributed_no_spawn: - start_rank = cfg.distributed_training.distributed_rank - cfg.distributed_training.distributed_rank = None # assign automatically - kwargs["start_rank"] = start_rank - torch.multiprocessing.spawn( - fn=distributed_main, - args=(main, cfg, kwargs), - nprocs=min( - torch.cuda.device_count(), - cfg.distributed_training.distributed_world_size, - ), - join=True, - ) - else: - distributed_main(cfg.distributed_training.device_id, main, cfg, kwargs) - elif cfg.common.tpu and cfg.distributed_training.distributed_world_size > 1: - import torch_xla.distributed.xla_multiprocessing as xmp - - torch.multiprocessing.set_sharing_strategy("file_system") - xmp.spawn( - fn=distributed_main, - args=(main, cfg, kwargs), - # tpu-comment: - # 8 devices in one TPU VM, is the max processes to be spawned. - # The rest is driven by xm.distributed.xla_dist - nprocs=min(cfg.distributed_training.distributed_world_size, 8), - ) - else: - # single GPU main - main(cfg, **kwargs) - - -def use_xla(): - global _USE_XLA - return _USE_XLA - - -def new_groups(grouped_ranks: List[List[int]]): - if use_xla(): - return ("tpu", grouped_ranks) - else: - groups = [dist.new_group(g) for g in grouped_ranks] - my_group_idx = _find_my_group_index(grouped_ranks) - return groups[my_group_idx] - - -def _find_my_group_index(grouped_ranks): - my_rank = get_global_rank() - for i, group in enumerate(grouped_ranks): - if my_rank in group: - return i - raise RuntimeError - - -def _find_my_group(grouped_ranks): - index = _find_my_group_index(grouped_ranks) - return grouped_ranks[index] - - -def get_rank(group): - if use_xla(): - assert group[0] == "tpu" - my_group = _find_my_group(group[1]) - return my_group.index(get_global_rank()) - else: - return dist.get_rank(group=group) - - -def get_world_size(group): - if use_xla(): - assert group[0] == "tpu" - my_group = _find_my_group(group[1]) - return len(my_group) - elif torch.distributed.is_initialized(): - return dist.get_world_size(group=group) - else: - return 1 - - -def get_global_group(): - if use_xla(): - return new_groups([list(range(get_global_world_size()))]) - elif torch.distributed.is_initialized(): - if not hasattr(get_global_group, "_global_group"): - # ideally we could use torch.distributed.group.WORLD, but it seems - # to cause random NCCL hangs in some cases - get_global_group._global_group = dist.new_group() - return get_global_group._global_group - else: - return None - - -def get_global_rank(): - if use_xla(): - return xm.get_ordinal() - elif torch.distributed.is_initialized(): - return torch.distributed.get_rank() - else: - return 0 - - -def get_global_world_size(): - if use_xla(): - return xm.xrt_world_size() - elif torch.distributed.is_initialized(): - return torch.distributed.get_world_size() - else: - return 1 - - -def get_data_parallel_group(): - """Get the data parallel group the caller rank belongs to.""" - global _USE_MEGATRON - if _USE_MEGATRON: - from fairseq.model_parallel.megatron import mpu - - return mpu.get_data_parallel_group() - else: - return get_global_group() - - -def get_data_parallel_rank(): - """Return my rank for the data parallel group.""" - return get_rank(get_data_parallel_group()) - - -def get_data_parallel_world_size(): - """Return world size for the data parallel group.""" - return get_world_size(get_data_parallel_group()) - - -def get_model_parallel_group(): - global _USE_MEGATRON - if _USE_MEGATRON: - from fairseq.model_parallel.megatron import mpu - - return mpu.get_model_parallel_group() - else: - return None - - -def get_model_parallel_rank(): - """Return my rank for the model parallel group.""" - return get_rank(get_model_parallel_group()) - - -def get_model_parallel_world_size(): - """Return world size for the model parallel group.""" - return get_world_size(get_model_parallel_group()) - - -def all_reduce(tensor, group, op="sum"): - if use_xla(): - assert isinstance(group, tuple) and group[0] == "tpu" - tensor = [tensor] # wrap in a list to make xm.all_reduce in-place - return xm.all_reduce(op, tensor, groups=group[1])[0] - else: - if op == "sum": - op = dist.ReduceOp.SUM - elif op == "max": - op = dist.ReduceOp.MAX - else: - raise NotImplementedError - dist.all_reduce(tensor, op=op, group=group) - return tensor - - -def broadcast(tensor, src, group): - if use_xla(): - # XLA doesn't support broadcast, hack it with all_reduce - if get_rank(group) != src: - tensor.zero_() - all_reduce(tensor, group) - else: - dist.broadcast(tensor, src=src, group=group) - - -def all_to_all(tensor, group): - """Perform an all-to-all operation on a 1D Tensor.""" - assert tensor.dim() == 1 - split_count = get_world_size(group=group) - assert tensor.numel() % split_count == 0 - if use_xla(): - assert isinstance(group, tuple) and group[0] == "tpu" - return xm.all_to_all( - tensor, - split_dimension=0, - concat_dimension=0, - split_count=split_count, - groups=group[1], - ) - else: - output = torch.zeros_like(tensor) - dist.all_to_all_single(output, tensor, group=group) - return output - - -def all_gather(tensor, group, return_tensor=False): - """Perform an all-gather operation.""" - if use_xla(): - result = xm.all_gather(tensor, groups=group[1]) - world_size = get_world_size(group=group) - result = result.view(world_size, *tensor.size()) - if return_tensor: - return result - else: - return [result[i] for i in range(world_size)] - else: - world_size = get_world_size(group=group) - rank = get_rank(group=group) - tensor_list = [ - tensor if i == rank else torch.empty_like(tensor) for i in range(world_size) - ] - dist.all_gather(tensor_list, tensor, group=group) - if return_tensor: - return torch.stack(tensor_list, dim=0) - else: - return tensor_list - - -def all_gather_list(data, group=None, max_size=16384): - """Gathers arbitrary data from all nodes into a list. - - Similar to :func:`~torch.distributed.all_gather` but for arbitrary Python - data. Note that *data* must be picklable and any CUDA tensors will be moved - to CPU and returned on CPU as well. - - Args: - data (Any): data from the local worker to be gathered on other workers - group: group of the collective - max_size (int, optional): maximum size of the data to be gathered - across workers - """ - from fairseq import utils - - if group is None: - group = get_global_group() - rank = get_rank(group=group) - world_size = get_world_size(group=group) - - buffer_size = max_size * world_size - if ( - not hasattr(all_gather_list, "_buffer") - or all_gather_list._buffer.numel() < buffer_size - ): - all_gather_list._buffer = torch.cuda.ByteTensor(buffer_size) - all_gather_list._cpu_buffer = torch.ByteTensor(max_size).pin_memory() - buffer = all_gather_list._buffer - buffer.zero_() - cpu_buffer = all_gather_list._cpu_buffer - - data = utils.move_to_cpu(data) - enc = pickle.dumps(data) - enc_size = len(enc) - header_size = 4 # size of header that contains the length of the encoded data - size = header_size + enc_size - if size > max_size: - raise ValueError( - "encoded data size ({}) exceeds max_size ({})".format(size, max_size) - ) - - header = struct.pack(">I", enc_size) - cpu_buffer[:size] = torch.ByteTensor(list(header + enc)) - start = rank * max_size - buffer[start : start + size].copy_(cpu_buffer[:size]) - - all_reduce(buffer, group=group) - - buffer = buffer.cpu() - try: - result = [] - for i in range(world_size): - out_buffer = buffer[i * max_size : (i + 1) * max_size] - (enc_size,) = struct.unpack(">I", bytes(out_buffer[:header_size].tolist())) - if enc_size > 0: - result.append( - pickle.loads( - bytes(out_buffer[header_size : header_size + enc_size].tolist()) - ) - ) - return result - except pickle.UnpicklingError: - raise Exception( - "Unable to unpickle data from other workers. all_gather_list requires all " - "workers to enter the function together, so this error usually indicates " - "that the workers have fallen out of sync somehow. Workers can fall out of " - "sync if one of them runs out of memory, or if there are other conditions " - "in your training script that can cause one worker to finish an epoch " - "while other workers are still iterating over their portions of the data. " - "Try rerunning with --ddp-backend=legacy_ddp and see if that helps." - ) - - -def all_reduce_dict(data: Mapping[str, Any], device, group) -> Dict[str, Any]: - """ - AllReduce a dictionary of values across workers. We separately - reduce items that are already on the device and items on CPU for - better performance. - - Args: - data (Mapping[str, Any]): dictionary of data to all-reduce, but - cannot be a nested dictionary - device (torch.device): device for the reduction - group: group of the collective - """ - data_keys = list(data.keys()) - - # We want to separately reduce items that are already on the - # device and items on CPU for performance reasons. - cpu_data = OrderedDict() - device_data = OrderedDict() - for k in data_keys: - t = data[k] - if not torch.is_tensor(t): - cpu_data[k] = torch.tensor(t, dtype=torch.double) - elif t.device.type != device.type: - cpu_data[k] = t.to(dtype=torch.double) - else: - device_data[k] = t.to(dtype=torch.double) - - def _all_reduce_dict(data: OrderedDict): - if len(data) == 0: - return data - buf = torch.cat([t.view(-1) for t in data.values()]).to(device=device) - all_reduce(buf, group=group) - split_buf = torch.split(buf.clone(), [t.numel() for t in data.values()]) - reduced_data = [t.view_as(orig) for t, orig in zip(split_buf, data.values())] - return OrderedDict(zip(data.keys(), reduced_data)) - - cpu_data = _all_reduce_dict(cpu_data) - device_data = _all_reduce_dict(device_data) - - def get_from_stack(key): - if key in cpu_data: - return cpu_data[key] - elif key in device_data: - return device_data[key] - raise KeyError - - return OrderedDict([(key, get_from_stack(key)) for key in data_keys]) - - -def broadcast_tensors( - tensors: Optional[List[torch.Tensor]], - src_rank: int, - group: object, - dist_device: Optional[torch.device] = None, -) -> List[torch.Tensor]: - """ - Broadcasts a list of tensors without other (non-src) ranks needing to know - the dtypes/shapes of the tensors. - """ - if dist_device is None: - if torch.distributed.get_backend(group) == "nccl": - dist_device = torch.device("cuda") - else: - dist_device = torch.device("cpu") - - # share metadata first to simplify transfer - is_src_rank = get_rank(group) == src_rank - if is_src_rank: - metadata = [ - {"size": t.size(), "dtype": t.dtype, "device": t.device} for t in tensors - ] - metadata = _broadcast_object_slow(metadata, src_rank, group, dist_device) - else: - metadata = _broadcast_object_slow(None, src_rank, group, dist_device) - - out_tensors = [] - for i, meta in enumerate(metadata): - if is_src_rank: - tensor = tensors[i] - broadcast(tensors[i].to(dist_device), src=src_rank, group=group) - else: - tensor = torch.zeros( - [meta["size"].numel()], dtype=meta["dtype"], device=dist_device - ) - broadcast(tensor, src=src_rank, group=group) - tensor = tensor.view(meta["size"]).to(meta["device"]) - out_tensors.append(tensor) - return out_tensors - - -def broadcast_object( - obj: Any, - src_rank: int, - group: object, - dist_device: Optional[torch.device] = None, -) -> Any: - """Broadcast an arbitrary Python object to other workers.""" - if dist_device is None: - if torch.distributed.get_backend(group) == "nccl": - dist_device = torch.device("cuda") - else: - dist_device = torch.device("cpu") - - if get_rank(group) == src_rank: - # split the tensors from the non-tensors so we can broadcast them - # directly, avoiding unnecessary serialization/deserialization - tensors = [] - obj = _split_tensors_from_obj(obj, tensors) - obj = _broadcast_object_slow(obj, src_rank, group, dist_device) - tensors = broadcast_tensors(tensors, src_rank, group, dist_device) - else: - obj = _broadcast_object_slow(None, src_rank, group, dist_device) - tensors = broadcast_tensors(None, src_rank, group, dist_device) - return _put_tensors_in_obj(obj, tensors) - - -def _broadcast_object_slow( - obj: Any, - src_rank: int, - group: object, - dist_device: torch.device, -) -> Any: - if get_rank(group) == src_rank: - # Emit data - buffer = io.BytesIO() - torch.save(obj, buffer) - buffer = torch.ByteTensor(buffer.getbuffer()).to(dist_device) - length = torch.LongTensor([len(buffer)]).to(dist_device) - broadcast(length, src=src_rank, group=group) - broadcast(buffer, src=src_rank, group=group) - else: - # Fetch from the source - length = torch.LongTensor([0]).to(dist_device) - broadcast(length, src=src_rank, group=group) - buffer = torch.ByteTensor(int(length.item())).to(dist_device) - broadcast(buffer, src=src_rank, group=group) - buffer = io.BytesIO(buffer.cpu().numpy()) - obj = torch.load(buffer, map_location="cpu") - return obj - - -@dataclass(frozen=True) -class _TensorPlaceholder: - index: int - - -def _split_tensors_from_obj(obj: Any, tensors: List[torch.Tensor]) -> Any: - if torch.is_tensor(obj): - placeholder = _TensorPlaceholder(index=len(tensors)) - tensors.append(obj) - return placeholder - elif isinstance(obj, dict): - return {k: _split_tensors_from_obj(v, tensors) for k, v in obj.items()} - elif isinstance(obj, list): - return [_split_tensors_from_obj(v, tensors) for v in obj] - elif isinstance(obj, tuple): - return tuple(_split_tensors_from_obj(v, tensors) for v in obj) - elif isinstance(obj, set): - return {_split_tensors_from_obj(v, tensors) for v in obj} - else: - return obj - - -def _put_tensors_in_obj(obj: Any, tensors: List[torch.Tensor]) -> Any: - if isinstance(obj, _TensorPlaceholder): - return tensors[obj.index] - elif isinstance(obj, dict): - return {k: _put_tensors_in_obj(v, tensors) for k, v in obj.items()} - elif isinstance(obj, list): - return [_put_tensors_in_obj(v, tensors) for v in obj] - elif isinstance(obj, tuple): - return tuple(_put_tensors_in_obj(v, tensors) for v in obj) - elif isinstance(obj, set): - return {_put_tensors_in_obj(v, tensors) for v in obj} - else: - return obj diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/file_chunker_utils.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/file_chunker_utils.py deleted file mode 100644 index 3f275490993dbbcd05d990c050ae6c7b4c9568c9..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/file_chunker_utils.py +++ /dev/null @@ -1,84 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import os -import typing as tp - - -def _safe_readline(fd) -> str: - pos = fd.tell() - while True: - try: - return fd.readline() - except UnicodeDecodeError: - pos -= 1 - fd.seek(pos) # search where this character begins - - -def find_offsets(filename: str, num_chunks: int) -> tp.List[int]: - """ - given a file and a number of chuncks, find the offsets in the file - to be able to chunk around full lines. - """ - with open(filename, "r", encoding="utf-8") as f: - size = os.fstat(f.fileno()).st_size - chunk_size = size // num_chunks - offsets = [0 for _ in range(num_chunks + 1)] - for i in range(1, num_chunks): - f.seek(chunk_size * i) - _safe_readline(f) - offsets[i] = f.tell() - offsets[-1] = size - return offsets - - -class ChunkLineIterator: - """ - Iterator to properly iterate over lines of a file chunck. - """ - - def __init__(self, fd, start_offset: int, end_offset: int): - self._fd = fd - self._start_offset = start_offset - self._end_offset = end_offset - - def __iter__(self) -> tp.Iterable[str]: - self._fd.seek(self._start_offset) - # next(f) breaks f.tell(), hence readline() must be used - line = _safe_readline(self._fd) - while line: - pos = self._fd.tell() - # f.tell() does not always give the byte position in the file - # sometimes it skips to a very large number - # it is unlikely that through a normal read we go from - # end bytes to end + 2**32 bytes (4 GB) and this makes it unlikely - # that the procedure breaks by the undeterministic behavior of - # f.tell() - if ( - self._end_offset > 0 - and pos > self._end_offset - and pos < self._end_offset + 2**32 - ): - break - yield line - line = self._fd.readline() - - -class Chunker: - """ - contextmanager to read a chunck of a file line by line. - """ - - def __init__(self, path: str, start_offset: int, end_offset: int): - self.path = path - self.start_offset = start_offset - self.end_offset = end_offset - - def __enter__(self) -> ChunkLineIterator: - self.fd = open(self.path, "r", encoding="utf-8") - return ChunkLineIterator(self.fd, self.start_offset, self.end_offset) - - def __exit__(self, exc_type, exc_val, exc_tb) -> None: - self.fd.close() diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/file_io.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/file_io.py deleted file mode 100644 index f3f37fe3f0c7d28bbeca216ddd93cb892b620cdc..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/file_io.py +++ /dev/null @@ -1,195 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import logging -import os -import shutil -from typing import List, Optional - -logger = logging.getLogger(__file__) - - -try: - from iopath.common.file_io import g_pathmgr as IOPathManager - - try: - # [FB only - for now] AWS PathHandler for PathManager - from .fb_pathhandlers import S3PathHandler - - IOPathManager.register_handler(S3PathHandler()) - except KeyError: - logging.warning("S3PathHandler already registered.") - except ImportError: - logging.debug( - "S3PathHandler couldn't be imported. Either missing fb-only files, or boto3 module." - ) - -except ImportError: - IOPathManager = None - - -class PathManager: - """ - Wrapper for insulating OSS I/O (using Python builtin operations) from - iopath's PathManager abstraction (for transparently handling various - internal backends). - """ - - @staticmethod - def open( - path: str, - mode: str = "r", - buffering: int = -1, - encoding: Optional[str] = None, - errors: Optional[str] = None, - newline: Optional[str] = None, - ): - if IOPathManager: - return IOPathManager.open( - path=path, - mode=mode, - buffering=buffering, - encoding=encoding, - errors=errors, - newline=newline, - ) - return open( - path, - mode=mode, - buffering=buffering, - encoding=encoding, - errors=errors, - newline=newline, - ) - - @staticmethod - def copy(src_path: str, dst_path: str, overwrite: bool = False) -> bool: - if IOPathManager: - return IOPathManager.copy( - src_path=src_path, dst_path=dst_path, overwrite=overwrite - ) - return shutil.copyfile(src_path, dst_path) - - @staticmethod - def get_local_path(path: str, **kwargs) -> str: - if IOPathManager: - return IOPathManager.get_local_path(path, **kwargs) - return path - - @staticmethod - def exists(path: str) -> bool: - if IOPathManager: - return IOPathManager.exists(path) - return os.path.exists(path) - - @staticmethod - def isfile(path: str) -> bool: - if IOPathManager: - return IOPathManager.isfile(path) - return os.path.isfile(path) - - @staticmethod - def ls(path: str) -> List[str]: - if IOPathManager: - return IOPathManager.ls(path) - return os.listdir(path) - - @staticmethod - def mkdirs(path: str) -> None: - if IOPathManager: - return IOPathManager.mkdirs(path) - os.makedirs(path, exist_ok=True) - - @staticmethod - def rm(path: str) -> None: - if IOPathManager: - return IOPathManager.rm(path) - os.remove(path) - - @staticmethod - def chmod(path: str, mode: int) -> None: - if not PathManager.path_requires_pathmanager(path): - os.chmod(path, mode) - - @staticmethod - def register_handler(handler) -> None: - if IOPathManager: - return IOPathManager.register_handler(handler=handler) - - @staticmethod - def copy_from_local( - local_path: str, dst_path: str, overwrite: bool = False, **kwargs - ) -> None: - if IOPathManager: - return IOPathManager.copy_from_local( - local_path=local_path, dst_path=dst_path, overwrite=overwrite, **kwargs - ) - return shutil.copyfile(local_path, dst_path) - - @staticmethod - def path_requires_pathmanager(path: str) -> bool: - """Do we require PathManager to access given path?""" - if IOPathManager: - for p in IOPathManager._path_handlers.keys(): - if path.startswith(p): - return True - return False - - @staticmethod - def supports_rename(path: str) -> bool: - # PathManager doesn't yet support renames - return not PathManager.path_requires_pathmanager(path) - - @staticmethod - def rename(src: str, dst: str): - os.rename(src, dst) - - """ - ioPath async PathManager methods: - """ - - @staticmethod - def opena( - path: str, - mode: str = "r", - buffering: int = -1, - encoding: Optional[str] = None, - errors: Optional[str] = None, - newline: Optional[str] = None, - ): - """ - Return file descriptor with asynchronous write operations. - """ - global IOPathManager - if not IOPathManager: - logging.info("ioPath is initializing PathManager.") - try: - from iopath.common.file_io import PathManager - - IOPathManager = PathManager() - except Exception: - logging.exception("Failed to initialize ioPath PathManager object.") - return IOPathManager.opena( - path=path, - mode=mode, - buffering=buffering, - encoding=encoding, - errors=errors, - newline=newline, - ) - - @staticmethod - def async_close() -> bool: - """ - Wait for files to be written and clean up asynchronous PathManager. - NOTE: `PathManager.async_close()` must be called at the end of any - script that uses `PathManager.opena(...)`. - """ - global IOPathManager - if IOPathManager: - return IOPathManager.async_close() - return False diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/file_utils.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/file_utils.py deleted file mode 100644 index 1c8883dca9ef185bd32ff8966614f4e25599e664..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/file_utils.py +++ /dev/null @@ -1,369 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -""" -Utilities for working with the local dataset cache. -This file is adapted from `AllenNLP <https://github.com/allenai/allennlp>`_. -and `huggingface <https://github.com/huggingface>`_. -""" - -import fnmatch -import json -import logging -import os -import shutil -import tarfile -import tempfile -from functools import partial, wraps -from hashlib import sha256 -from io import open - -try: - from torch.hub import _get_torch_home - - torch_cache_home = _get_torch_home() -except ImportError: - torch_cache_home = os.path.expanduser( - os.getenv( - "TORCH_HOME", os.path.join(os.getenv("XDG_CACHE_HOME", "~/.cache"), "torch") - ) - ) -default_cache_path = os.path.join(torch_cache_home, "pytorch_fairseq") - -try: - from urllib.parse import urlparse -except ImportError: - from urlparse import urlparse - -try: - from pathlib import Path - - PYTORCH_FAIRSEQ_CACHE = Path(os.getenv("PYTORCH_FAIRSEQ_CACHE", default_cache_path)) -except (AttributeError, ImportError): - PYTORCH_FAIRSEQ_CACHE = os.getenv("PYTORCH_FAIRSEQ_CACHE", default_cache_path) - -CONFIG_NAME = "config.json" -WEIGHTS_NAME = "pytorch_model.bin" - -logger = logging.getLogger(__name__) # pylint: disable=invalid-name - - -def load_archive_file(archive_file): - # redirect to the cache, if necessary - try: - resolved_archive_file = cached_path(archive_file, cache_dir=None) - except EnvironmentError: - logger.info( - "Archive name '{}' was not found in archive name list. " - "We assumed '{}' was a path or URL but couldn't find any file " - "associated to this path or URL.".format( - archive_file, - archive_file, - ) - ) - return None - - if resolved_archive_file == archive_file: - logger.info("loading archive file {}".format(archive_file)) - else: - logger.info( - "loading archive file {} from cache at {}".format( - archive_file, resolved_archive_file - ) - ) - - # Extract archive to temp dir and replace .tar.bz2 if necessary - tempdir = None - if not os.path.isdir(resolved_archive_file): - tempdir = tempfile.mkdtemp() - logger.info( - "extracting archive file {} to temp dir {}".format( - resolved_archive_file, tempdir - ) - ) - ext = os.path.splitext(archive_file)[1][1:] - with tarfile.open(resolved_archive_file, "r:" + ext) as archive: - top_dir = os.path.commonprefix(archive.getnames()) - archive.extractall(tempdir) - os.remove(resolved_archive_file) - shutil.move(os.path.join(tempdir, top_dir), resolved_archive_file) - shutil.rmtree(tempdir) - - return resolved_archive_file - - -def url_to_filename(url, etag=None): - """ - Convert `url` into a hashed filename in a repeatable way. - If `etag` is specified, append its hash to the URL's, delimited - by a period. - """ - url_bytes = url.encode("utf-8") - url_hash = sha256(url_bytes) - filename = url_hash.hexdigest() - - if etag: - etag_bytes = etag.encode("utf-8") - etag_hash = sha256(etag_bytes) - filename += "." + etag_hash.hexdigest() - - return filename - - -def filename_to_url(filename, cache_dir=None): - """ - Return the url and etag (which may be ``None``) stored for `filename`. - Raise ``EnvironmentError`` if `filename` or its stored metadata do not exist. - """ - if cache_dir is None: - cache_dir = PYTORCH_FAIRSEQ_CACHE - if isinstance(cache_dir, Path): - cache_dir = str(cache_dir) - - cache_path = os.path.join(cache_dir, filename) - if not os.path.exists(cache_path): - raise EnvironmentError("file {} not found".format(cache_path)) - - meta_path = cache_path + ".json" - if not os.path.exists(meta_path): - raise EnvironmentError("file {} not found".format(meta_path)) - - with open(meta_path, encoding="utf-8") as meta_file: - metadata = json.load(meta_file) - url = metadata["url"] - etag = metadata["etag"] - - return url, etag - - -def cached_path_from_pm(url_or_filename): - """ - Tries to cache the specified URL using PathManager class. - Returns the cached path if success otherwise failure. - """ - try: - from fairseq.file_io import PathManager - - local_path = PathManager.get_local_path(url_or_filename) - return local_path - except Exception: - return None - - -def cached_path(url_or_filename, cache_dir=None): - """ - Given something that might be a URL (or might be a local path), - determine which. If it's a URL, download the file and cache it, and - return the path to the cached file. If it's already a local path, - make sure the file exists and then return the path. - """ - if cache_dir is None: - cache_dir = PYTORCH_FAIRSEQ_CACHE - if isinstance(url_or_filename, Path): - url_or_filename = str(url_or_filename) - if isinstance(cache_dir, Path): - cache_dir = str(cache_dir) - - parsed = urlparse(url_or_filename) - - if parsed.scheme in ("http", "https", "s3"): - # URL, so get it from the cache (downloading if necessary) - return get_from_cache(url_or_filename, cache_dir) - elif os.path.exists(url_or_filename): - # File, and it exists. - return url_or_filename - elif parsed.scheme == "": - # File, but it doesn't exist. - raise EnvironmentError("file {} not found".format(url_or_filename)) - else: - cached_path = cached_path_from_pm(url_or_filename) - if cached_path: - return cached_path - # Something unknown - raise ValueError( - "unable to parse {} as a URL or as a local path".format(url_or_filename) - ) - - -def split_s3_path(url): - """Split a full s3 path into the bucket name and path.""" - parsed = urlparse(url) - if not parsed.netloc or not parsed.path: - raise ValueError("bad s3 path {}".format(url)) - bucket_name = parsed.netloc - s3_path = parsed.path - # Remove '/' at beginning of path. - if s3_path.startswith("/"): - s3_path = s3_path[1:] - return bucket_name, s3_path - - -def s3_request(func): - """ - Wrapper function for s3 requests in order to create more helpful error - messages. - """ - - @wraps(func) - def wrapper(url, *args, **kwargs): - from botocore.exceptions import ClientError - - try: - return func(url, *args, **kwargs) - except ClientError as exc: - if int(exc.response["Error"]["Code"]) == 404: - raise EnvironmentError("file {} not found".format(url)) - else: - raise - - return wrapper - - -@s3_request -def s3_etag(url): - """Check ETag on S3 object.""" - import boto3 - - s3_resource = boto3.resource("s3") - bucket_name, s3_path = split_s3_path(url) - s3_object = s3_resource.Object(bucket_name, s3_path) - return s3_object.e_tag - - -@s3_request -def s3_get(url, temp_file): - """Pull a file directly from S3.""" - import boto3 - - s3_resource = boto3.resource("s3") - bucket_name, s3_path = split_s3_path(url) - s3_resource.Bucket(bucket_name).download_fileobj(s3_path, temp_file) - - -def request_wrap_timeout(func, url): - import requests - - for attempt, timeout in enumerate([10, 20, 40, 60, 60]): - try: - return func(timeout=timeout) - except requests.exceptions.Timeout as e: - logger.warning( - "Request for %s timed-out (attempt %d). Retrying with a timeout of %d secs", - url, - attempt, - timeout, - exc_info=e, - ) - continue - raise RuntimeError(f"Unable to fetch file {url}") - - -def http_get(url, temp_file): - import requests - from tqdm import tqdm - - req = request_wrap_timeout(partial(requests.get, url, stream=True), url) - content_length = req.headers.get("Content-Length") - total = int(content_length) if content_length is not None else None - progress = tqdm(unit="B", total=total) - for chunk in req.iter_content(chunk_size=1024): - if chunk: # filter out keep-alive new chunks - progress.update(len(chunk)) - temp_file.write(chunk) - progress.close() - - -def get_from_cache(url, cache_dir=None): - """ - Given a URL, look for the corresponding dataset in the local cache. - If it's not there, download it. Then return the path to the cached file. - """ - if cache_dir is None: - cache_dir = PYTORCH_FAIRSEQ_CACHE - if isinstance(cache_dir, Path): - cache_dir = str(cache_dir) - - if not os.path.exists(cache_dir): - os.makedirs(cache_dir) - - # Get eTag to add to filename, if it exists. - if url.startswith("s3://"): - etag = s3_etag(url) - else: - try: - import requests - - response = request_wrap_timeout( - partial(requests.head, url, allow_redirects=True), url - ) - if response.status_code != 200: - etag = None - else: - etag = response.headers.get("ETag") - except RuntimeError: - etag = None - - filename = url_to_filename(url, etag) - - # get cache path to put the file - cache_path = os.path.join(cache_dir, filename) - - # If we don't have a connection (etag is None) and can't identify the file - # try to get the last downloaded one - if not os.path.exists(cache_path) and etag is None: - matching_files = fnmatch.filter(os.listdir(cache_dir), filename + ".*") - matching_files = list(filter(lambda s: not s.endswith(".json"), matching_files)) - if matching_files: - cache_path = os.path.join(cache_dir, matching_files[-1]) - - if not os.path.exists(cache_path): - # Download to temporary file, then copy to cache dir once finished. - # Otherwise you get corrupt cache entries if the download gets interrupted. - with tempfile.NamedTemporaryFile() as temp_file: - logger.info("%s not found in cache, downloading to %s", url, temp_file.name) - - # GET file object - if url.startswith("s3://"): - s3_get(url, temp_file) - else: - http_get(url, temp_file) - - # we are copying the file before closing it, so flush to avoid truncation - temp_file.flush() - # shutil.copyfileobj() starts at the current position, so go to the start - temp_file.seek(0) - - logger.info("copying %s to cache at %s", temp_file.name, cache_path) - with open(cache_path, "wb") as cache_file: - shutil.copyfileobj(temp_file, cache_file) - - logger.info("creating metadata file for %s", cache_path) - meta = {"url": url, "etag": etag} - meta_path = cache_path + ".json" - with open(meta_path, "w") as meta_file: - output_string = json.dumps(meta) - meta_file.write(output_string) - - logger.info("removing temp file %s", temp_file.name) - - return cache_path - - -def read_set_from_file(filename): - """ - Extract a de-duped collection (set) of text from a file. - Expected file format is one item per line. - """ - collection = set() - with open(filename, "r", encoding="utf-8") as file_: - for line in file_: - collection.add(line.rstrip()) - return collection - - -def get_file_extension(path, dot=True, lower=True): - ext = os.path.splitext(path)[1] - ext = ext if dot else ext[1:] - return ext.lower() if lower else ext diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/hub_utils.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/hub_utils.py deleted file mode 100644 index 0b9bbb611a7b1278be91081f49cd987652871498..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/hub_utils.py +++ /dev/null @@ -1,313 +0,0 @@ -#!/usr/bin/env python3 -u -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import copy -import logging -import os -from typing import Any, Dict, Iterator, List - -import torch -from fairseq import utils -from fairseq.data import encoders -from omegaconf import open_dict -from torch import nn - -logger = logging.getLogger(__name__) - - -def from_pretrained( - model_name_or_path, - checkpoint_file="model.pt", - data_name_or_path=".", - archive_map=None, - **kwargs -): - from fairseq import checkpoint_utils, file_utils - - if archive_map is not None: - if model_name_or_path in archive_map: - model_name_or_path = archive_map[model_name_or_path] - if data_name_or_path is not None and data_name_or_path in archive_map: - data_name_or_path = archive_map[data_name_or_path] - - # allow archive_map to set default arg_overrides (e.g., tokenizer, bpe) - # for each model - if isinstance(model_name_or_path, dict): - for k, v in model_name_or_path.items(): - if k == "checkpoint_file": - checkpoint_file = v - elif ( - k != "path" - # only set kwargs that don't already have overrides - and k not in kwargs - ): - kwargs[k] = v - model_name_or_path = model_name_or_path["path"] - - model_path = file_utils.load_archive_file(model_name_or_path) - - # convenience hack for loading data and BPE codes from model archive - if data_name_or_path.startswith("."): - kwargs["data"] = os.path.abspath(os.path.join(model_path, data_name_or_path)) - else: - kwargs["data"] = file_utils.load_archive_file(data_name_or_path) - for file, arg in { - "code": "bpe_codes", - "bpecodes": "bpe_codes", - "sentencepiece.bpe.model": "sentencepiece_model", - "merges.txt": "bpe_merges", - "vocab.json": "bpe_vocab", - }.items(): - path = os.path.join(model_path, file) - if os.path.exists(path): - kwargs[arg] = path - - if "user_dir" in kwargs: - utils.import_user_module(argparse.Namespace(user_dir=kwargs["user_dir"])) - - models, args, task = checkpoint_utils.load_model_ensemble_and_task( - [os.path.join(model_path, cpt) for cpt in checkpoint_file.split(os.pathsep)], - arg_overrides=kwargs, - ) - - return { - "args": args, - "task": task, - "models": models, - } - - -class GeneratorHubInterface(nn.Module): - """ - PyTorch Hub interface for generating sequences from a pre-trained - translation or language model. - """ - - def __init__(self, cfg, task, models): - super().__init__() - self.cfg = cfg - self.task = task - self.models = nn.ModuleList(models) - self.src_dict = task.source_dictionary - self.tgt_dict = task.target_dictionary - - # optimize model for generation - for model in self.models: - model.prepare_for_inference_(cfg) - - # Load alignment dictionary for unknown word replacement - # (None if no unknown word replacement, empty if no path to align dictionary) - self.align_dict = utils.load_align_dict(cfg.generation.replace_unk) - - self.tokenizer = encoders.build_tokenizer(cfg.tokenizer) - self.bpe = encoders.build_bpe(cfg.bpe) - - self.max_positions = utils.resolve_max_positions( - self.task.max_positions(), *[model.max_positions() for model in models] - ) - - # this is useful for determining the device - self.register_buffer("_float_tensor", torch.tensor([0], dtype=torch.float)) - - @property - def device(self): - return self._float_tensor.device - - def translate( - self, sentences: List[str], beam: int = 5, verbose: bool = False, **kwargs - ) -> List[str]: - return self.sample(sentences, beam, verbose, **kwargs) - - def sample( - self, sentences: List[str], beam: int = 1, verbose: bool = False, **kwargs - ) -> List[str]: - if isinstance(sentences, str): - return self.sample([sentences], beam=beam, verbose=verbose, **kwargs)[0] - tokenized_sentences = [self.encode(sentence) for sentence in sentences] - batched_hypos = self.generate(tokenized_sentences, beam, verbose, **kwargs) - return [self.decode(hypos[0]["tokens"]) for hypos in batched_hypos] - - def score( - self, sentences: List[str], replace_newline_with_eos: bool = False, **kwargs - ): - if isinstance(sentences, str): - return self.score( - [sentences], replace_newline_with_eos=replace_newline_with_eos, **kwargs - )[0] - - def encode(sentence): - if replace_newline_with_eos: - return torch.cat([self.encode(line) for line in sentence.splitlines()]) - else: - return self.encode(sentence) - - # NOTE: this doesn't support translation tasks currently - tokenized_sentences = [encode(sentence) for sentence in sentences] - return [ - hypos[0] - for hypos in self.generate( - tokenized_sentences, score_reference=True, **kwargs - ) - ] - - def generate( - self, - tokenized_sentences: List[torch.LongTensor], - beam: int = 5, - verbose: bool = False, - skip_invalid_size_inputs=False, - inference_step_args=None, - prefix_allowed_tokens_fn=None, - **kwargs - ) -> List[List[Dict[str, torch.Tensor]]]: - if torch.is_tensor(tokenized_sentences) and tokenized_sentences.dim() == 1: - return self.generate( - tokenized_sentences.unsqueeze(0), beam=beam, verbose=verbose, **kwargs - )[0] - - # build generator using current args as well as any kwargs - gen_args = copy.deepcopy(self.cfg.generation) - with open_dict(gen_args): - gen_args.beam = beam - for k, v in kwargs.items(): - setattr(gen_args, k, v) - generator = self.task.build_generator( - self.models, - gen_args, - prefix_allowed_tokens_fn=prefix_allowed_tokens_fn, - ) - - inference_step_args = inference_step_args or {} - results = [] - for batch in self._build_batches(tokenized_sentences, skip_invalid_size_inputs): - batch = utils.apply_to_sample(lambda t: t.to(self.device), batch) - translations = self.task.inference_step( - generator, self.models, batch, **inference_step_args - ) - for id, hypos in zip(batch["id"].tolist(), translations): - results.append((id, hypos)) - - # sort output to match input order - outputs = [hypos for _, hypos in sorted(results, key=lambda x: x[0])] - - if verbose: - - def getarg(name, default): - return getattr(gen_args, name, getattr(self.cfg, name, default)) - - for source_tokens, target_hypotheses in zip(tokenized_sentences, outputs): - src_str_with_unk = self.string(source_tokens) - logger.info("S\t{}".format(src_str_with_unk)) - for hypo in target_hypotheses: - hypo_str = self.decode(hypo["tokens"]) - logger.info("H\t{}\t{}".format(hypo["score"], hypo_str)) - logger.info( - "P\t{}".format( - " ".join( - map( - lambda x: "{:.4f}".format(x), - hypo["positional_scores"].tolist(), - ) - ) - ) - ) - if hypo["alignment"] is not None and getarg( - "print_alignment", False - ): - logger.info( - "A\t{}".format( - " ".join( - [ - "{}-{}".format(src_idx, tgt_idx) - for src_idx, tgt_idx in hypo["alignment"] - ] - ) - ) - ) - return outputs - - def encode(self, sentence: str) -> torch.LongTensor: - sentence = self.tokenize(sentence) - sentence = self.apply_bpe(sentence) - return self.binarize(sentence) - - def decode(self, tokens: torch.LongTensor) -> str: - sentence = self.string(tokens) - sentence = self.remove_bpe(sentence) - return self.detokenize(sentence) - - def tokenize(self, sentence: str) -> str: - if self.tokenizer is not None: - sentence = self.tokenizer.encode(sentence) - return sentence - - def detokenize(self, sentence: str) -> str: - if self.tokenizer is not None: - sentence = self.tokenizer.decode(sentence) - return sentence - - def apply_bpe(self, sentence: str) -> str: - if self.bpe is not None: - sentence = self.bpe.encode(sentence) - return sentence - - def remove_bpe(self, sentence: str) -> str: - if self.bpe is not None: - sentence = self.bpe.decode(sentence) - return sentence - - def binarize(self, sentence: str) -> torch.LongTensor: - return self.src_dict.encode_line(sentence, add_if_not_exist=False).long() - - def string(self, tokens: torch.LongTensor) -> str: - return self.tgt_dict.string(tokens) - - def _build_batches( - self, tokens: List[List[int]], skip_invalid_size_inputs: bool - ) -> Iterator[Dict[str, Any]]: - lengths = torch.LongTensor([t.numel() for t in tokens]) - batch_iterator = self.task.get_batch_iterator( - dataset=self.task.build_dataset_for_inference(tokens, lengths), - max_tokens=self.cfg.dataset.max_tokens, - max_sentences=self.cfg.dataset.batch_size, - max_positions=self.max_positions, - ignore_invalid_inputs=skip_invalid_size_inputs, - disable_iterator_cache=True, - ).next_epoch_itr(shuffle=False) - return batch_iterator - - -class BPEHubInterface(object): - """PyTorch Hub interface for Byte-Pair Encoding (BPE).""" - - def __init__(self, bpe, **kwargs): - super().__init__() - args = argparse.Namespace(bpe=bpe, **kwargs) - self.bpe = encoders.build_bpe(args) - assert self.bpe is not None - - def encode(self, sentence: str) -> str: - return self.bpe.encode(sentence) - - def decode(self, sentence: str) -> str: - return self.bpe.decode(sentence) - - -class TokenizerHubInterface(object): - """PyTorch Hub interface for tokenization.""" - - def __init__(self, tokenizer, **kwargs): - super().__init__() - args = argparse.Namespace(tokenizer=tokenizer, **kwargs) - self.tokenizer = encoders.build_tokenizer(args) - assert self.tokenizer is not None - - def encode(self, sentence: str) -> str: - return self.tokenizer.encode(sentence) - - def decode(self, sentence: str) -> str: - return self.tokenizer.decode(sentence) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/incremental_decoding_utils.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/incremental_decoding_utils.py deleted file mode 100644 index b26e6cd01cd4cbdffa23d88b354eb4a55a94189b..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/incremental_decoding_utils.py +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import uuid -from typing import Dict, Optional - -from torch import Tensor - - -class FairseqIncrementalState(object): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.init_incremental_state() - - def init_incremental_state(self): - self._incremental_state_id = str(uuid.uuid4()) - - def _get_full_incremental_state_key(self, key: str) -> str: - return "{}.{}".format(self._incremental_state_id, key) - - def get_incremental_state( - self, - incremental_state: Optional[Dict[str, Dict[str, Optional[Tensor]]]], - key: str, - ) -> Optional[Dict[str, Optional[Tensor]]]: - """Helper for getting incremental state for an nn.Module.""" - full_key = self._get_full_incremental_state_key(key) - if incremental_state is None or full_key not in incremental_state: - return None - return incremental_state[full_key] - - def set_incremental_state( - self, - incremental_state: Optional[Dict[str, Dict[str, Optional[Tensor]]]], - key: str, - value: Dict[str, Optional[Tensor]], - ) -> Optional[Dict[str, Dict[str, Optional[Tensor]]]]: - """Helper for setting incremental state for an nn.Module.""" - if incremental_state is not None: - full_key = self._get_full_incremental_state_key(key) - incremental_state[full_key] = value - return incremental_state - - -def with_incremental_state(cls): - cls.__bases__ = (FairseqIncrementalState,) + tuple( - b for b in cls.__bases__ if b != FairseqIncrementalState - ) - return cls diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/iterative_refinement_generator.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/iterative_refinement_generator.py deleted file mode 100644 index b62d8c626d57ff49b40b6cf8164b5878b7b28870..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/iterative_refinement_generator.py +++ /dev/null @@ -1,358 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from collections import namedtuple - -import numpy as np -import torch -from fairseq import utils - -DecoderOut = namedtuple( - "IterativeRefinementDecoderOut", - ["output_tokens", "output_scores", "attn", "step", "max_step", "history"], -) - - -class IterativeRefinementGenerator(object): - def __init__( - self, - tgt_dict, - models=None, - eos_penalty=0.0, - max_iter=10, - max_ratio=2, - beam_size=1, - decoding_format=None, - retain_dropout=False, - adaptive=True, - retain_history=False, - reranking=False, - ): - """ - Generates translations based on iterative refinement. - - Args: - tgt_dict: target dictionary - eos_penalty: if > 0.0, it penalized early-stopping in decoding - max_iter: maximum number of refinement iterations - max_ratio: generate sequences of maximum length ax, where x is the source length - decoding_format: decoding mode in {'unigram', 'ensemble', 'vote', 'dp', 'bs'} - retain_dropout: retaining dropout in the inference - adaptive: decoding with early stop - """ - self.bos = tgt_dict.bos() - self.pad = tgt_dict.pad() - self.unk = tgt_dict.unk() - self.eos = tgt_dict.eos() - self.vocab_size = len(tgt_dict) - self.eos_penalty = eos_penalty - self.max_iter = max_iter - self.max_ratio = max_ratio - self.beam_size = beam_size - self.reranking = reranking - self.decoding_format = decoding_format - self.retain_dropout = retain_dropout - self.retain_history = retain_history - self.adaptive = adaptive - self.models = models - - def generate_batched_itr( - self, - data_itr, - maxlen_a=None, - maxlen_b=None, - cuda=False, - timer=None, - prefix_size=0, - ): - """Iterate over a batched dataset and yield individual translations. - - Args: - maxlen_a/b: generate sequences of maximum length ax + b, - where x is the source sentence length. - cuda: use GPU for generation - timer: StopwatchMeter for timing generations. - """ - - for sample in data_itr: - if "net_input" not in sample: - continue - if timer is not None: - timer.start() - with torch.no_grad(): - hypos = self.generate( - self.models, - sample, - prefix_tokens=sample["target"][:, :prefix_size] - if prefix_size > 0 - else None, - ) - if timer is not None: - timer.stop(sample["ntokens"]) - for i, id in enumerate(sample["id"]): - # remove padding - src = utils.strip_pad(sample["net_input"]["src_tokens"][i, :], self.pad) - ref = utils.strip_pad(sample["target"][i, :], self.pad) - yield id, src, ref, hypos[i] - - @torch.no_grad() - def generate(self, models, sample, prefix_tokens=None, constraints=None): - if constraints is not None: - raise NotImplementedError( - "Constrained decoding with the IterativeRefinementGenerator is not supported" - ) - - # TODO: iterative refinement generator does not support ensemble for now. - if not self.retain_dropout: - for model in models: - model.eval() - - model, reranker = models[0], None - if self.reranking: - assert len(models) > 1, "Assuming the last checkpoint is the reranker" - assert ( - self.beam_size > 1 - ), "Reranking requires multiple translation for each example" - - reranker = models[-1] - models = models[:-1] - - if len(models) > 1 and hasattr(model, "enable_ensemble"): - assert model.allow_ensemble, "{} does not support ensembling".format( - model.__class__.__name__ - ) - model.enable_ensemble(models) - - # TODO: better encoder inputs? - src_tokens = sample["net_input"]["src_tokens"] - src_lengths = sample["net_input"]["src_lengths"] - bsz, src_len = src_tokens.size() - - # initialize - encoder_out = model.forward_encoder([src_tokens, src_lengths]) - prev_decoder_out = model.initialize_output_tokens(encoder_out, src_tokens) - - if self.beam_size > 1: - assert ( - model.allow_length_beam - ), "{} does not support decoding with length beam.".format( - model.__class__.__name__ - ) - - # regenerate data based on length-beam - length_beam_order = ( - utils.new_arange(src_tokens, self.beam_size, bsz).t().reshape(-1) - ) - encoder_out = model.encoder.reorder_encoder_out( - encoder_out, length_beam_order - ) - prev_decoder_out = model.regenerate_length_beam( - prev_decoder_out, self.beam_size - ) - bsz = bsz * self.beam_size - - sent_idxs = torch.arange(bsz) - prev_output_tokens = prev_decoder_out.output_tokens.clone() - - if self.retain_history: - prev_decoder_out = prev_decoder_out._replace(history=[prev_output_tokens]) - - finalized = [[] for _ in range(bsz)] - - def is_a_loop(x, y, s, a): - b, l_x, l_y = x.size(0), x.size(1), y.size(1) - if l_x > l_y: - y = torch.cat([y, x.new_zeros(b, l_x - l_y).fill_(self.pad)], 1) - s = torch.cat([s, s.new_zeros(b, l_x - l_y)], 1) - if a is not None: - a = torch.cat([a, a.new_zeros(b, l_x - l_y, a.size(2))], 1) - elif l_x < l_y: - x = torch.cat([x, y.new_zeros(b, l_y - l_x).fill_(self.pad)], 1) - return (x == y).all(1), y, s, a - - def finalized_hypos(step, prev_out_token, prev_out_score, prev_out_attn): - cutoff = prev_out_token.ne(self.pad) - tokens = prev_out_token[cutoff] - if prev_out_score is None: - scores, score = None, None - else: - scores = prev_out_score[cutoff] - score = scores.mean() - - if prev_out_attn is None: - hypo_attn, alignment = None, None - else: - hypo_attn = prev_out_attn[cutoff] - alignment = hypo_attn.max(dim=1)[1] - return { - "steps": step, - "tokens": tokens, - "positional_scores": scores, - "score": score, - "hypo_attn": hypo_attn, - "alignment": alignment, - } - - for step in range(self.max_iter + 1): - - decoder_options = { - "eos_penalty": self.eos_penalty, - "max_ratio": self.max_ratio, - "decoding_format": self.decoding_format, - } - prev_decoder_out = prev_decoder_out._replace( - step=step, - max_step=self.max_iter + 1, - ) - - decoder_out = model.forward_decoder( - prev_decoder_out, encoder_out, **decoder_options - ) - - if self.adaptive: - # terminate if there is a loop - terminated, out_tokens, out_scores, out_attn = is_a_loop( - prev_output_tokens, - decoder_out.output_tokens, - decoder_out.output_scores, - decoder_out.attn, - ) - decoder_out = decoder_out._replace( - output_tokens=out_tokens, - output_scores=out_scores, - attn=out_attn, - ) - - else: - terminated = decoder_out.output_tokens.new_zeros( - decoder_out.output_tokens.size(0) - ).bool() - - if step == self.max_iter: # reach last iteration, terminate - terminated.fill_(1) - - # collect finalized sentences - finalized_idxs = sent_idxs[terminated] - finalized_tokens = decoder_out.output_tokens[terminated] - finalized_scores = decoder_out.output_scores[terminated] - finalized_attn = ( - None - if (decoder_out.attn is None or decoder_out.attn.size(0) == 0) - else decoder_out.attn[terminated] - ) - - if self.retain_history: - finalized_history_tokens = [h[terminated] for h in decoder_out.history] - - for i in range(finalized_idxs.size(0)): - finalized[finalized_idxs[i]] = [ - finalized_hypos( - step, - finalized_tokens[i], - finalized_scores[i], - None if finalized_attn is None else finalized_attn[i], - ) - ] - - if self.retain_history: - finalized[finalized_idxs[i]][0]["history"] = [] - for j in range(len(finalized_history_tokens)): - finalized[finalized_idxs[i]][0]["history"].append( - finalized_hypos( - step, finalized_history_tokens[j][i], None, None - ) - ) - - # check if all terminated - if terminated.sum() == terminated.size(0): - break - - # for next step - not_terminated = ~terminated - prev_decoder_out = decoder_out._replace( - output_tokens=decoder_out.output_tokens[not_terminated], - output_scores=decoder_out.output_scores[not_terminated], - attn=decoder_out.attn[not_terminated] - if (decoder_out.attn is not None and decoder_out.attn.size(0) > 0) - else None, - history=[h[not_terminated] for h in decoder_out.history] - if decoder_out.history is not None - else None, - ) - encoder_out = model.encoder.reorder_encoder_out( - encoder_out, not_terminated.nonzero(as_tuple=False).squeeze() - ) - sent_idxs = sent_idxs[not_terminated] - prev_output_tokens = prev_decoder_out.output_tokens.clone() - - if self.beam_size > 1: - if reranker is not None: - finalized = self.rerank( - reranker, finalized, [src_tokens, src_lengths], self.beam_size - ) - - # aggregate information from length beam - finalized = [ - finalized[ - np.argmax( - [ - finalized[self.beam_size * i + j][0]["score"] - for j in range(self.beam_size) - ] - ) - + self.beam_size * i - ] - for i in range(len(finalized) // self.beam_size) - ] - - return finalized - - def rerank(self, reranker, finalized, encoder_input, beam_size): - def rebuild_batch(finalized): - finalized_tokens = [f[0]["tokens"] for f in finalized] - finalized_maxlen = max(f.size(0) for f in finalized_tokens) - final_output_tokens = ( - finalized_tokens[0] - .new_zeros(len(finalized_tokens), finalized_maxlen) - .fill_(self.pad) - ) - for i, f in enumerate(finalized_tokens): - final_output_tokens[i, : f.size(0)] = f - return final_output_tokens - - final_output_tokens = rebuild_batch(finalized) - final_output_tokens[ - :, 0 - ] = self.eos # autoregressive model assumes starting with EOS - - reranker_encoder_out = reranker.encoder(*encoder_input) - length_beam_order = ( - utils.new_arange( - final_output_tokens, beam_size, reranker_encoder_out.encoder_out.size(1) - ) - .t() - .reshape(-1) - ) - reranker_encoder_out = reranker.encoder.reorder_encoder_out( - reranker_encoder_out, length_beam_order - ) - reranking_scores = reranker.get_normalized_probs( - reranker.decoder(final_output_tokens[:, :-1], reranker_encoder_out), - True, - None, - ) - reranking_scores = reranking_scores.gather(2, final_output_tokens[:, 1:, None]) - reranking_masks = final_output_tokens[:, 1:].ne(self.pad) - reranking_scores = ( - reranking_scores[:, :, 0].masked_fill_(~reranking_masks, 0).sum(1) - ) - reranking_scores = reranking_scores / reranking_masks.sum(1).type_as( - reranking_scores - ) - - for i in range(len(finalized)): - finalized[i][0]["score"] = reranking_scores[i] - - return finalized diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/logging/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/logging/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/logging/meters.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/logging/meters.py deleted file mode 100644 index d5f7c775d94a9ae56ba815e486e0ce535001dc52..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/logging/meters.py +++ /dev/null @@ -1,321 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import bisect -import time -from collections import OrderedDict -from typing import Dict, Optional - -try: - import torch - - def type_as(a, b): - if torch.is_tensor(a) and torch.is_tensor(b): - return a.to(b) - else: - return a - -except ImportError: - torch = None - - def type_as(a, b): - return a - - -try: - import numpy as np -except ImportError: - np = None - - -class Meter(object): - """Base class for Meters.""" - - def __init__(self): - pass - - def state_dict(self): - return {} - - def load_state_dict(self, state_dict): - pass - - def reset(self): - raise NotImplementedError - - @property - def smoothed_value(self) -> float: - """Smoothed value used for logging.""" - raise NotImplementedError - - -def safe_round(number, ndigits): - if hasattr(number, "__round__"): - return round(number, ndigits) - elif torch is not None and torch.is_tensor(number) and number.numel() == 1: - return safe_round(number.item(), ndigits) - elif np is not None and np.ndim(number) == 0 and hasattr(number, "item"): - return safe_round(number.item(), ndigits) - else: - return number - - -class AverageMeter(Meter): - """Computes and stores the average and current value""" - - def __init__(self, round: Optional[int] = None): - self.round = round - self.reset() - - def reset(self): - self.val = None # most recent update - self.sum = 0 # sum from all updates - self.count = 0 # total n from all updates - - def update(self, val, n=1): - if val is not None: - self.val = val - if n > 0: - self.sum = type_as(self.sum, val) + (val * n) - self.count = type_as(self.count, n) + n - - def state_dict(self): - return { - "val": self.val, - "sum": self.sum, - "count": self.count, - "round": self.round, - } - - def load_state_dict(self, state_dict): - self.val = state_dict["val"] - self.sum = state_dict["sum"] - self.count = state_dict["count"] - self.round = state_dict.get("round", None) - - @property - def avg(self): - return self.sum / self.count if self.count > 0 else self.val - - @property - def smoothed_value(self) -> float: - val = self.avg - if self.round is not None and val is not None: - val = safe_round(val, self.round) - return val - - -class SumMeter(Meter): - """Computes and stores the sum""" - - def __init__(self, round: Optional[int] = None): - self.round = round - self.reset() - - def reset(self): - self.sum = 0 # sum from all updates - - def update(self, val): - if val is not None: - self.sum = type_as(self.sum, val) + val - - def state_dict(self): - return { - "sum": self.sum, - "round": self.round, - } - - def load_state_dict(self, state_dict): - self.sum = state_dict["sum"] - self.round = state_dict.get("round", None) - - @property - def smoothed_value(self) -> float: - val = self.sum - if self.round is not None and val is not None: - val = safe_round(val, self.round) - return val - - -class TimeMeter(Meter): - """Computes the average occurrence of some event per second""" - - def __init__( - self, - init: int = 0, - n: int = 0, - round: Optional[int] = None, - ): - self.round = round - self.reset(init, n) - - def reset(self, init=0, n=0): - self.init = init - self.start = time.perf_counter() - self.n = n - self.i = 0 - - def update(self, val=1): - self.n = type_as(self.n, val) + val - self.i += 1 - - def state_dict(self): - return { - "init": self.elapsed_time, - "n": self.n, - "round": self.round, - } - - def load_state_dict(self, state_dict): - if "start" in state_dict: - # backwards compatibility for old state_dicts - self.reset(init=state_dict["init"]) - else: - self.reset(init=state_dict["init"], n=state_dict["n"]) - self.round = state_dict.get("round", None) - - @property - def avg(self): - return self.n / self.elapsed_time - - @property - def elapsed_time(self): - return self.init + (time.perf_counter() - self.start) - - @property - def smoothed_value(self) -> float: - val = self.avg - if self.round is not None and val is not None: - val = safe_round(val, self.round) - return val - - -class StopwatchMeter(Meter): - """Computes the sum/avg duration of some event in seconds""" - - def __init__(self, round: Optional[int] = None): - self.round = round - self.sum = 0 - self.n = 0 - self.start_time = None - - def start(self): - self.start_time = time.perf_counter() - - def stop(self, n=1, prehook=None): - if self.start_time is not None: - if prehook is not None: - prehook() - delta = time.perf_counter() - self.start_time - self.sum = self.sum + delta - self.n = type_as(self.n, n) + n - - def reset(self): - self.sum = 0 # cumulative time during which stopwatch was active - self.n = 0 # total n across all start/stop - self.start() - - def state_dict(self): - return { - "sum": self.sum, - "n": self.n, - "round": self.round, - } - - def load_state_dict(self, state_dict): - self.sum = state_dict["sum"] - self.n = state_dict["n"] - self.start_time = None - self.round = state_dict.get("round", None) - - @property - def avg(self): - return self.sum / self.n if self.n > 0 else self.sum - - @property - def elapsed_time(self): - if self.start_time is None: - return 0.0 - return time.perf_counter() - self.start_time - - @property - def smoothed_value(self) -> float: - val = self.avg if self.sum > 0 else self.elapsed_time - if self.round is not None and val is not None: - val = safe_round(val, self.round) - return val - - -class MetersDict(OrderedDict): - """A sorted dictionary of :class:`Meters`. - - Meters are sorted according to a priority that is given when the - meter is first added to the dictionary. - """ - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.priorities = [] - - def __setitem__(self, key, value): - assert key not in self, "MetersDict doesn't support reassignment" - priority, value = value - bisect.insort(self.priorities, (priority, len(self.priorities), key)) - super().__setitem__(key, value) - for _, _, key in self.priorities: # reorder dict to match priorities - self.move_to_end(key) - - def add_meter(self, key, meter, priority): - self.__setitem__(key, (priority, meter)) - - def state_dict(self): - return [ - (pri, key, self[key].__class__.__name__, self[key].state_dict()) - for pri, _, key in self.priorities - # can't serialize DerivedMeter instances - if not isinstance(self[key], MetersDict._DerivedMeter) - ] - - def load_state_dict(self, state_dict): - self.clear() - self.priorities.clear() - for pri, key, meter_cls, meter_state in state_dict: - meter = globals()[meter_cls]() - meter.load_state_dict(meter_state) - self.add_meter(key, meter, pri) - - def get_smoothed_value(self, key: str) -> float: - """Get a single smoothed value.""" - meter = self[key] - if isinstance(meter, MetersDict._DerivedMeter): - return meter.fn(self) - else: - return meter.smoothed_value - - def get_smoothed_values(self) -> Dict[str, float]: - """Get all smoothed values.""" - return OrderedDict( - [ - (key, self.get_smoothed_value(key)) - for key in self.keys() - if not key.startswith("_") - ] - ) - - def reset(self): - """Reset Meter instances.""" - for meter in self.values(): - if isinstance(meter, MetersDict._DerivedMeter): - continue - meter.reset() - - class _DerivedMeter(Meter): - """A Meter whose values are derived from other Meters.""" - - def __init__(self, fn): - self.fn = fn - - def reset(self): - pass diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/logging/metrics.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/logging/metrics.py deleted file mode 100644 index 466d6821ecf8c744407b402c79f9a1283b66a495..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/logging/metrics.py +++ /dev/null @@ -1,315 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. -""" -A standalone module for aggregating metrics. - -Metrics can be logged from anywhere using the `log_*` functions defined -in this module. The logged values will be aggregated dynamically based -on the aggregation context in which the logging occurs. See the -:func:`aggregate` context manager for more details. -""" - -import contextlib -import uuid -from collections import defaultdict -from typing import Callable, List, Optional - -from .meters import * - -# Aggregation contexts are considered "active" when inside the scope -# created by the :func:`aggregate` context manager. -_aggregators = OrderedDict() -_active_aggregators = OrderedDict() -_active_aggregators_cnt = defaultdict(lambda: 0) - - -def reset() -> None: - """Reset all metrics aggregators.""" - _aggregators.clear() - _active_aggregators.clear() - _active_aggregators_cnt.clear() - - # The "default" aggregator observes all logged values. - _aggregators["default"] = MetersDict() - _active_aggregators["default"] = _aggregators["default"] - _active_aggregators_cnt["default"] = 1 - - -reset() - - -@contextlib.contextmanager -def aggregate(name: Optional[str] = None, new_root: bool = False): - """Context manager to aggregate metrics under a given name. - - Aggregations can be nested. If *new_root* is ``False``, then logged - metrics will be recorded along the entire stack of nested - aggregators, including a global "default" aggregator. If *new_root* - is ``True``, then this aggregator will be the root of a new - aggregation stack, thus bypassing any parent aggregators. - - Note that aggregation contexts are uniquely identified by their - *name* (e.g., train, valid). Creating a context with an existing - name will reuse the corresponding :class:`MetersDict` instance. - If no name is given, then a temporary aggregator will be created. - - Usage:: - - with metrics.aggregate("train"): - for step, batch in enumerate(epoch): - with metrics.aggregate("train_inner") as agg: - metrics.log_scalar("loss", get_loss(batch)) - if step % log_interval == 0: - print(agg.get_smoothed_value("loss")) - agg.reset() - print(metrics.get_smoothed_values("train")["loss"]) - - Args: - name (str): name of the aggregation. Defaults to a - random/temporary name if not given explicitly. - new_root (bool): make this aggregation the root of a new - aggregation stack. - """ - if name is None: - # generate a temporary name - name = str(uuid.uuid4()) - assert name not in _aggregators - agg = MetersDict() - else: - assert name != "default" - agg = _aggregators.setdefault(name, MetersDict()) - - if new_root: - backup_aggregators = _active_aggregators.copy() - _active_aggregators.clear() - backup_aggregators_cnt = _active_aggregators_cnt.copy() - _active_aggregators_cnt.clear() - - _active_aggregators[name] = agg - _active_aggregators_cnt[name] += 1 - - yield agg - - _active_aggregators_cnt[name] -= 1 - if _active_aggregators_cnt[name] == 0 and name in _active_aggregators: - del _active_aggregators[name] - - if new_root: - _active_aggregators.clear() - _active_aggregators.update(backup_aggregators) - _active_aggregators_cnt.clear() - _active_aggregators_cnt.update(backup_aggregators_cnt) - - -def get_active_aggregators() -> List[MetersDict]: - return list(_active_aggregators.values()) - - -def log_scalar( - key: str, - value: float, - weight: float = 1, - priority: int = 10, - round: Optional[int] = None, -): - """Log a scalar value. - - Args: - key (str): name of the field to log - value (float): value to log - weight (float): weight that this value contributes to the average. - A weight of 0 will always log the latest value. - priority (int): smaller values are logged earlier in the output - round (Optional[int]): number of digits to round to when displaying - """ - for agg in get_active_aggregators(): - if key not in agg: - agg.add_meter(key, AverageMeter(round=round), priority) - agg[key].update(value, weight) - - -def log_scalar_sum( - key: str, - value: float, - priority: int = 10, - round: Optional[int] = None, -): - """Log a scalar value that is summed for reporting. - - Args: - key (str): name of the field to log - value (float): value to log - priority (int): smaller values are logged earlier in the output - round (Optional[int]): number of digits to round to when displaying - """ - for agg in get_active_aggregators(): - if key not in agg: - agg.add_meter(key, SumMeter(round=round), priority) - agg[key].update(value) - - -def log_derived(key: str, fn: Callable[[MetersDict], float], priority: int = 20): - """Log a scalar value derived from other meters. - - Args: - key (str): name of the field to log - fn (Callable[[MetersDict], float]): function that takes a single - argument *meters* and returns the derived value - priority (int): smaller values are logged earlier in the output - """ - for agg in get_active_aggregators(): - if key not in agg: - agg.add_meter(key, MetersDict._DerivedMeter(fn), priority) - - -def log_speed( - key: str, - value: float, - priority: int = 30, - round: Optional[int] = None, -): - """Log the rate of some quantity per second. - - Args: - key (str): name of the field to log - value (float): value to log - priority (int): smaller values are logged earlier in the output - round (Optional[int]): number of digits to round to when displaying - """ - for agg in get_active_aggregators(): - if key not in agg: - agg.add_meter(key, TimeMeter(round=round), priority) - agg[key].reset() # reset meter on the first call - else: - agg[key].update(value) - - -def log_start_time(key: str, priority: int = 40, round: Optional[int] = None): - """Log the duration of some event in seconds. - - The duration will be computed once :func:`log_stop_time` is called. - - Args: - key (str): name of the field to log - priority (int): smaller values are logged earlier in the output - round (Optional[int]): number of digits to round to when displaying - """ - for agg in get_active_aggregators(): - if key not in agg: - agg.add_meter(key, StopwatchMeter(round=round), priority) - agg[key].start() - - -def log_stop_time(key: str, weight: float = 0.0, prehook=None): - """Log the duration of some event in seconds. - - The duration will be computed since :func:`log_start_time` was called. - Set weight > 0 to report the average time instead of the sum. - - Args: - key (str): name of the field to log - weight (float): weight that this time contributes to the average - prehook (function, no arguments): will be called before the timer - is stopped. For example, use prehook=torch.cuda.synchronize to - make sure all gpu operations are done before timer is stopped. - """ - for agg in get_active_aggregators(): - if key in agg: - agg[key].stop(weight, prehook) - - -def log_custom( - new_meter_fn: Callable[[], Meter], - key: str, - *args, - priority: int = 50, - **kwargs, -): - """Log using a custom Meter. - - Any extra *args* or *kwargs* will be passed through to the Meter's - *update* method. - - Args: - new_meter_fn (Callable[[], Meter]): function that returns a new - Meter instance - key (str): name of the field to log - priority (int): smaller values are logged earlier in the output - """ - for agg in get_active_aggregators(): - if key not in agg: - agg.add_meter(key, new_meter_fn(), priority) - agg[key].update(*args, **kwargs) - - -def reset_meter(name: str, key: str) -> None: - """Reset Meter instance aggregated under a given *name* and *key*.""" - meter = get_meter(name, key) - if meter is not None: - meter.reset() - - -def reset_meters(name: str) -> None: - """Reset Meter instances aggregated under a given *name*.""" - meters = get_meters(name) - if meters is not None: - meters.reset() - - -def get_meter(name: str, key: str) -> Meter: - """Get a single Meter instance aggregated under *name* and *key*. - - Returns: - Meter or None if no metrics have been logged under *name* and *key*. - """ - if name not in _aggregators: - return None - return _aggregators[name].get(key, None) - - -def get_meters(name: str) -> MetersDict: - """Get Meter instances aggregated under a given *name*. - - Returns: - MetersDict or None if no metrics have been logged under *name*. - """ - return _aggregators.get(name, None) - - -def get_smoothed_value(name: str, key: str) -> float: - """Get a single smoothed value. - - Raises: - KeyError: if no metrics have been logged under *name* and *key*. - """ - return _aggregators[name].get_smoothed_value(key) - - -def get_smoothed_values(name: str) -> Dict[str, float]: - """Get smoothed values aggregated under a given *name*. - - Raises: - KeyError: if no metrics have been logged under *name*. - """ - return _aggregators[name].get_smoothed_values() - - -def state_dict(): - return OrderedDict([(name, agg.state_dict()) for name, agg in _aggregators.items()]) - - -def load_state_dict(state_dict): - for name, agg_state in state_dict.items(): - _aggregators[name] = MetersDict() - _aggregators[name].load_state_dict(agg_state) - - -def xla_metrics_report(): - try: - import torch_xla.debug.metrics as met - - print(met.metrics_report()) - except ImportError: - return diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/logging/progress_bar.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/logging/progress_bar.py deleted file mode 100644 index 4c64b61bad6edbf4b9ff5bcc2f26952e8b1bfc9c..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/logging/progress_bar.py +++ /dev/null @@ -1,582 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -""" -Wrapper around various loggers and progress bars (e.g., tqdm). -""" - -import atexit -import json -import logging -import os -import sys -from collections import OrderedDict -from contextlib import contextmanager -from numbers import Number -from typing import Optional - -import torch - -from .meters import AverageMeter, StopwatchMeter, TimeMeter - -logger = logging.getLogger(__name__) - - -def progress_bar( - iterator, - log_format: Optional[str] = None, - log_interval: int = 100, - log_file: Optional[str] = None, - epoch: Optional[int] = None, - prefix: Optional[str] = None, - aim_repo: Optional[str] = None, - aim_run_hash: Optional[str] = None, - aim_param_checkpoint_dir: Optional[str] = None, - tensorboard_logdir: Optional[str] = None, - default_log_format: str = "tqdm", - wandb_project: Optional[str] = None, - wandb_run_name: Optional[str] = None, - azureml_logging: Optional[bool] = False, -): - if log_format is None: - log_format = default_log_format - if log_file is not None: - handler = logging.FileHandler(filename=log_file) - logger.addHandler(handler) - - if log_format == "tqdm" and not sys.stderr.isatty(): - log_format = "simple" - - if log_format == "json": - bar = JsonProgressBar(iterator, epoch, prefix, log_interval) - elif log_format == "none": - bar = NoopProgressBar(iterator, epoch, prefix) - elif log_format == "simple": - bar = SimpleProgressBar(iterator, epoch, prefix, log_interval) - elif log_format == "tqdm": - bar = TqdmProgressBar(iterator, epoch, prefix) - else: - raise ValueError("Unknown log format: {}".format(log_format)) - - if aim_repo: - bar = AimProgressBarWrapper( - bar, - aim_repo=aim_repo, - aim_run_hash=aim_run_hash, - aim_param_checkpoint_dir=aim_param_checkpoint_dir, - ) - - if tensorboard_logdir: - try: - # [FB only] custom wrapper for TensorBoard - import palaas # noqa - - from .fb_tbmf_wrapper import FbTbmfWrapper - - bar = FbTbmfWrapper(bar, log_interval) - except ImportError: - bar = TensorboardProgressBarWrapper(bar, tensorboard_logdir) - - if wandb_project: - bar = WandBProgressBarWrapper(bar, wandb_project, run_name=wandb_run_name) - - if azureml_logging: - bar = AzureMLProgressBarWrapper(bar) - - return bar - - -def build_progress_bar( - args, - iterator, - epoch: Optional[int] = None, - prefix: Optional[str] = None, - default: str = "tqdm", - no_progress_bar: str = "none", -): - """Legacy wrapper that takes an argparse.Namespace.""" - if getattr(args, "no_progress_bar", False): - default = no_progress_bar - if getattr(args, "distributed_rank", 0) == 0: - tensorboard_logdir = getattr(args, "tensorboard_logdir", None) - else: - tensorboard_logdir = None - return progress_bar( - iterator, - log_format=args.log_format, - log_interval=args.log_interval, - epoch=epoch, - prefix=prefix, - tensorboard_logdir=tensorboard_logdir, - default_log_format=default, - ) - - -def format_stat(stat): - if isinstance(stat, Number): - stat = "{:g}".format(stat) - elif isinstance(stat, AverageMeter): - stat = "{:.3f}".format(stat.avg) - elif isinstance(stat, TimeMeter): - stat = "{:g}".format(round(stat.avg)) - elif isinstance(stat, StopwatchMeter): - stat = "{:g}".format(round(stat.sum)) - elif torch.is_tensor(stat): - stat = stat.tolist() - return stat - - -class BaseProgressBar(object): - """Abstract class for progress bars.""" - - def __init__(self, iterable, epoch=None, prefix=None): - self.iterable = iterable - self.n = getattr(iterable, "n", 0) - self.epoch = epoch - self.prefix = "" - if epoch is not None: - self.prefix += "epoch {:03d}".format(epoch) - if prefix is not None: - self.prefix += (" | " if self.prefix != "" else "") + prefix - - def __len__(self): - return len(self.iterable) - - def __enter__(self): - return self - - def __exit__(self, *exc): - return False - - def __iter__(self): - raise NotImplementedError - - def log(self, stats, tag=None, step=None): - """Log intermediate stats according to log_interval.""" - raise NotImplementedError - - def print(self, stats, tag=None, step=None): - """Print end-of-epoch stats.""" - raise NotImplementedError - - def update_config(self, config): - """Log latest configuration.""" - pass - - def _str_commas(self, stats): - return ", ".join(key + "=" + stats[key].strip() for key in stats.keys()) - - def _str_pipes(self, stats): - return " | ".join(key + " " + stats[key].strip() for key in stats.keys()) - - def _format_stats(self, stats): - postfix = OrderedDict(stats) - # Preprocess stats according to datatype - for key in postfix.keys(): - postfix[key] = str(format_stat(postfix[key])) - return postfix - - -@contextmanager -def rename_logger(logger, new_name): - old_name = logger.name - if new_name is not None: - logger.name = new_name - yield logger - logger.name = old_name - - -class JsonProgressBar(BaseProgressBar): - """Log output in JSON format.""" - - def __init__(self, iterable, epoch=None, prefix=None, log_interval=1000): - super().__init__(iterable, epoch, prefix) - self.log_interval = log_interval - self.i = None - self.size = None - - def __iter__(self): - self.size = len(self.iterable) - for i, obj in enumerate(self.iterable, start=self.n): - self.i = i - yield obj - - def log(self, stats, tag=None, step=None): - """Log intermediate stats according to log_interval.""" - step = step or self.i or 0 - if step > 0 and self.log_interval is not None and step % self.log_interval == 0: - update = ( - self.epoch - 1 + (self.i + 1) / float(self.size) - if self.epoch is not None - else None - ) - stats = self._format_stats(stats, epoch=self.epoch, update=update) - with rename_logger(logger, tag): - logger.info(json.dumps(stats)) - - def print(self, stats, tag=None, step=None): - """Print end-of-epoch stats.""" - self.stats = stats - if tag is not None: - self.stats = OrderedDict( - [(tag + "_" + k, v) for k, v in self.stats.items()] - ) - stats = self._format_stats(self.stats, epoch=self.epoch) - with rename_logger(logger, tag): - logger.info(json.dumps(stats)) - - def _format_stats(self, stats, epoch=None, update=None): - postfix = OrderedDict() - if epoch is not None: - postfix["epoch"] = epoch - if update is not None: - postfix["update"] = round(update, 3) - # Preprocess stats according to datatype - for key in stats.keys(): - postfix[key] = format_stat(stats[key]) - return postfix - - -class NoopProgressBar(BaseProgressBar): - """No logging.""" - - def __init__(self, iterable, epoch=None, prefix=None): - super().__init__(iterable, epoch, prefix) - - def __iter__(self): - for obj in self.iterable: - yield obj - - def log(self, stats, tag=None, step=None): - """Log intermediate stats according to log_interval.""" - pass - - def print(self, stats, tag=None, step=None): - """Print end-of-epoch stats.""" - pass - - -class SimpleProgressBar(BaseProgressBar): - """A minimal logger for non-TTY environments.""" - - def __init__(self, iterable, epoch=None, prefix=None, log_interval=1000): - super().__init__(iterable, epoch, prefix) - self.log_interval = log_interval - self.i = None - self.size = None - - def __iter__(self): - self.size = len(self.iterable) - for i, obj in enumerate(self.iterable, start=self.n): - self.i = i - yield obj - - def log(self, stats, tag=None, step=None): - """Log intermediate stats according to log_interval.""" - step = step or self.i or 0 - if step > 0 and self.log_interval is not None and step % self.log_interval == 0: - stats = self._format_stats(stats) - postfix = self._str_commas(stats) - with rename_logger(logger, tag): - logger.info( - "{}: {:5d} / {:d} {}".format( - self.prefix, self.i + 1, self.size, postfix - ) - ) - - def print(self, stats, tag=None, step=None): - """Print end-of-epoch stats.""" - postfix = self._str_pipes(self._format_stats(stats)) - with rename_logger(logger, tag): - logger.info("{} | {}".format(self.prefix, postfix)) - - -class TqdmProgressBar(BaseProgressBar): - """Log to tqdm.""" - - def __init__(self, iterable, epoch=None, prefix=None): - super().__init__(iterable, epoch, prefix) - from tqdm import tqdm - - self.tqdm = tqdm( - iterable, - self.prefix, - leave=False, - disable=(logger.getEffectiveLevel() > logging.INFO), - ) - - def __iter__(self): - return iter(self.tqdm) - - def log(self, stats, tag=None, step=None): - """Log intermediate stats according to log_interval.""" - self.tqdm.set_postfix(self._format_stats(stats), refresh=False) - - def print(self, stats, tag=None, step=None): - """Print end-of-epoch stats.""" - postfix = self._str_pipes(self._format_stats(stats)) - with rename_logger(logger, tag): - logger.info("{} | {}".format(self.prefix, postfix)) - - -try: - import functools - - from aim import Repo as AimRepo - - @functools.lru_cache() - def get_aim_run(repo, run_hash): - from aim import Run - - return Run(run_hash=run_hash, repo=repo) - -except ImportError: - get_aim_run = None - AimRepo = None - - -class AimProgressBarWrapper(BaseProgressBar): - """Log to Aim.""" - - def __init__(self, wrapped_bar, aim_repo, aim_run_hash, aim_param_checkpoint_dir): - self.wrapped_bar = wrapped_bar - - if get_aim_run is None: - self.run = None - logger.warning("Aim not found, please install with: pip install aim") - else: - logger.info(f"Storing logs at Aim repo: {aim_repo}") - - if not aim_run_hash: - # Find run based on save_dir parameter - query = f"run.checkpoint.save_dir == '{aim_param_checkpoint_dir}'" - try: - runs_generator = AimRepo(aim_repo).query_runs(query) - run = next(runs_generator.iter_runs()) - aim_run_hash = run.run.hash - except Exception: - pass - - if aim_run_hash: - logger.info(f"Appending to run: {aim_run_hash}") - - self.run = get_aim_run(aim_repo, aim_run_hash) - - def __iter__(self): - return iter(self.wrapped_bar) - - def log(self, stats, tag=None, step=None): - """Log intermediate stats to Aim.""" - self._log_to_aim(stats, tag, step) - self.wrapped_bar.log(stats, tag=tag, step=step) - - def print(self, stats, tag=None, step=None): - """Print end-of-epoch stats.""" - self._log_to_aim(stats, tag, step) - self.wrapped_bar.print(stats, tag=tag, step=step) - - def update_config(self, config): - """Log latest configuration.""" - if self.run is not None: - for key in config: - self.run.set(key, config[key], strict=False) - self.wrapped_bar.update_config(config) - - def _log_to_aim(self, stats, tag=None, step=None): - if self.run is None: - return - - if step is None: - step = stats["num_updates"] - - if "train" in tag: - context = {"tag": tag, "subset": "train"} - elif "val" in tag: - context = {"tag": tag, "subset": "val"} - else: - context = {"tag": tag} - - for key in stats.keys() - {"num_updates"}: - self.run.track(stats[key], name=key, step=step, context=context) - - -try: - _tensorboard_writers = {} - from torch.utils.tensorboard import SummaryWriter -except ImportError: - try: - from tensorboardX import SummaryWriter - except ImportError: - SummaryWriter = None - - -def _close_writers(): - for w in _tensorboard_writers.values(): - w.close() - - -atexit.register(_close_writers) - - -class TensorboardProgressBarWrapper(BaseProgressBar): - """Log to tensorboard.""" - - def __init__(self, wrapped_bar, tensorboard_logdir): - self.wrapped_bar = wrapped_bar - self.tensorboard_logdir = tensorboard_logdir - - if SummaryWriter is None: - logger.warning( - "tensorboard not found, please install with: pip install tensorboard" - ) - - def _writer(self, key): - if SummaryWriter is None: - return None - _writers = _tensorboard_writers - if key not in _writers: - _writers[key] = SummaryWriter(os.path.join(self.tensorboard_logdir, key)) - _writers[key].add_text("sys.argv", " ".join(sys.argv)) - return _writers[key] - - def __iter__(self): - return iter(self.wrapped_bar) - - def log(self, stats, tag=None, step=None): - """Log intermediate stats to tensorboard.""" - self._log_to_tensorboard(stats, tag, step) - self.wrapped_bar.log(stats, tag=tag, step=step) - - def print(self, stats, tag=None, step=None): - """Print end-of-epoch stats.""" - self._log_to_tensorboard(stats, tag, step) - self.wrapped_bar.print(stats, tag=tag, step=step) - - def update_config(self, config): - """Log latest configuration.""" - # TODO add hparams to Tensorboard - self.wrapped_bar.update_config(config) - - def _log_to_tensorboard(self, stats, tag=None, step=None): - writer = self._writer(tag or "") - if writer is None: - return - if step is None: - step = stats["num_updates"] - for key in stats.keys() - {"num_updates"}: - if isinstance(stats[key], AverageMeter): - writer.add_scalar(key, stats[key].val, step) - elif isinstance(stats[key], Number): - writer.add_scalar(key, stats[key], step) - elif torch.is_tensor(stats[key]) and stats[key].numel() == 1: - writer.add_scalar(key, stats[key].item(), step) - writer.flush() - - -try: - import wandb -except ImportError: - wandb = None - - -class WandBProgressBarWrapper(BaseProgressBar): - """Log to Weights & Biases.""" - - def __init__(self, wrapped_bar, wandb_project, run_name=None): - self.wrapped_bar = wrapped_bar - if wandb is None: - logger.warning("wandb not found, pip install wandb") - return - - # reinit=False to ensure if wandb.init() is called multiple times - # within one process it still references the same run - wandb.init(project=wandb_project, reinit=False, name=run_name) - - def __iter__(self): - return iter(self.wrapped_bar) - - def log(self, stats, tag=None, step=None): - """Log intermediate stats to tensorboard.""" - self._log_to_wandb(stats, tag, step) - self.wrapped_bar.log(stats, tag=tag, step=step) - - def print(self, stats, tag=None, step=None): - """Print end-of-epoch stats.""" - self._log_to_wandb(stats, tag, step) - self.wrapped_bar.print(stats, tag=tag, step=step) - - def update_config(self, config): - """Log latest configuration.""" - if wandb is not None: - wandb.config.update(config) - self.wrapped_bar.update_config(config) - - def _log_to_wandb(self, stats, tag=None, step=None): - if wandb is None: - return - if step is None: - step = stats["num_updates"] - - prefix = "" if tag is None else tag + "/" - - for key in stats.keys() - {"num_updates"}: - if isinstance(stats[key], AverageMeter): - wandb.log({prefix + key: stats[key].val}, step=step) - elif isinstance(stats[key], Number): - wandb.log({prefix + key: stats[key]}, step=step) - - -try: - from azureml.core import Run -except ImportError: - Run = None - - -class AzureMLProgressBarWrapper(BaseProgressBar): - """Log to Azure ML""" - - def __init__(self, wrapped_bar): - self.wrapped_bar = wrapped_bar - if Run is None: - logger.warning("azureml.core not found, pip install azureml-core") - return - self.run = Run.get_context() - - def __exit__(self, *exc): - if Run is not None: - self.run.complete() - return False - - def __iter__(self): - return iter(self.wrapped_bar) - - def log(self, stats, tag=None, step=None): - """Log intermediate stats to AzureML""" - self._log_to_azureml(stats, tag, step) - self.wrapped_bar.log(stats, tag=tag, step=step) - - def print(self, stats, tag=None, step=None): - """Print end-of-epoch stats""" - self._log_to_azureml(stats, tag, step) - self.wrapped_bar.print(stats, tag=tag, step=step) - - def update_config(self, config): - """Log latest configuration.""" - self.wrapped_bar.update_config(config) - - def _log_to_azureml(self, stats, tag=None, step=None): - if Run is None: - return - if step is None: - step = stats["num_updates"] - - prefix = "" if tag is None else tag + "/" - - for key in stats.keys() - {"num_updates"}: - name = prefix + key - if isinstance(stats[key], AverageMeter): - self.run.log_row(name=name, **{"step": step, key: stats[key].val}) - elif isinstance(stats[key], Number): - self.run.log_row(name=name, **{"step": step, key: stats[key]}) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/model_parallel/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/model_parallel/__init__.py deleted file mode 100644 index 69f21684872f72ae8ee26d9ff7d2d2b6e6d526c3..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/model_parallel/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from . import criterions, models, modules # noqa diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/model_parallel/criterions/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/model_parallel/criterions/__init__.py deleted file mode 100644 index 50b379a48444e32c10558550b33a68c9c0a97134..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/model_parallel/criterions/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import importlib -import os - -# automatically import any Python files in the criterions/ directory -for file in sorted(os.listdir(os.path.dirname(__file__))): - if file.endswith(".py") and not file.startswith("_"): - module = file[: file.find(".py")] - importlib.import_module("fairseq.model_parallel.criterions." + module) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/model_parallel/criterions/vocab_parallel_cross_entropy.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/model_parallel/criterions/vocab_parallel_cross_entropy.py deleted file mode 100644 index 75102cf012b24ac651f4775fec09937c5dea117d..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/model_parallel/criterions/vocab_parallel_cross_entropy.py +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import math - -from fairseq import metrics, utils -from fairseq.criterions import FairseqCriterion, register_criterion - -try: - from fairseq.model_parallel.megatron.mpu.cross_entropy import ( - vocab_parallel_cross_entropy, - ) - - has_megatron_submodule = True -except (ImportError, ModuleNotFoundError): - has_megatron_submodule = False - - -@register_criterion("vocab_parallel_cross_entropy") -class VocabParallelCrossEntropyCriterion(FairseqCriterion): - def __init__(self, task, sentence_avg): - super().__init__(task) - self.sentence_avg = sentence_avg - if not has_megatron_submodule: - raise ImportError( - "\n\nPlease install the megatron submodule:" - "\n\n git submodule update --init " - "fairseq/model_parallel/megatron" - ) - - def forward(self, model, sample, reduce=True): - """Compute the loss for the given sample. - - Returns a tuple with three elements: - 1) the loss - 2) the sample size, which is used as the denominator for the gradient - 3) logging outputs to display while training - """ - net_output = model(**sample["net_input"]) - target = sample["target"] - - loss = vocab_parallel_cross_entropy(net_output[0].float(), target) - loss = (loss * (target != self.padding_idx)).sum() - sample_size = ( - sample["target"].size(0) if self.sentence_avg else sample["ntokens"] - ) - logging_output = { - "loss": utils.item(loss.data) if reduce else loss.data, - "ntokens": sample["ntokens"], - "nsentences": sample["target"].size(0), - "sample_size": sample_size, - } - return loss, sample_size, logging_output - - @staticmethod - def reduce_metrics(logging_outputs) -> None: - """Aggregate logging outputs from data parallel training.""" - loss_sum = sum(log.get("loss", 0) for log in logging_outputs) - ntokens = sum(log.get("ntokens", 0) for log in logging_outputs) - sample_size = sum(log.get("sample_size", 0) for log in logging_outputs) - - metrics.log_scalar( - "loss", loss_sum / sample_size / math.log(2), sample_size, round=3 - ) - if sample_size != ntokens: - metrics.log_scalar( - "nll_loss", loss_sum / ntokens / math.log(2), ntokens, round=3 - ) - metrics.log_derived( - "ppl", lambda meters: utils.get_perplexity(meters["nll_loss"].avg) - ) - else: - metrics.log_derived( - "ppl", lambda meters: utils.get_perplexity(meters["loss"].avg) - ) - - @staticmethod - def logging_outputs_can_be_summed() -> bool: - """ - Whether the logging outputs returned by `forward` can be summed - across workers prior to calling `reduce_metrics`. Setting this - to True will improves distributed training speed. - """ - return True diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/model_parallel/megatron_trainer.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/model_parallel/megatron_trainer.py deleted file mode 100644 index 37431686af1697d7e8d4fa434cc585cfdcb5b0a3..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/model_parallel/megatron_trainer.py +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -""" -Train a network across multiple GPUs. -""" - -from fairseq.dataclass.configs import FairseqConfig -from fairseq.distributed import utils as distributed_utils -from fairseq.trainer import Trainer - -try: - from fairseq.model_parallel.megatron.mpu import ( - get_cuda_rng_tracker, - get_data_parallel_rank, - get_data_parallel_world_size, - get_model_parallel_src_rank, - ) - - has_megatron_submodule = True -except (ImportError, ModuleNotFoundError): - has_megatron_submodule = False - - -class MegatronTrainer(Trainer): - """Main class for model parallel with data parallel training.""" - - def __init__(self, cfg: FairseqConfig, task, model, criterion, **kwargs): - if not has_megatron_submodule: - raise ImportError( - "\n\nPlease install the megatron submodule:" - "\n\n git submodule update --init " - "fairseq/model_parallel/megatron" - ) - super().__init__(cfg, task, model, criterion, **kwargs) - - def clip_grad_norm(self, clip_norm): - def _aggregate_model_parallel_grad_norm(total_norm): - total_norm = total_norm ** 2 - distributed_utils.all_reduce( - total_norm, group=distributed_utils.get_model_parallel_group() - ) - total_norm = total_norm ** 0.5 - return total_norm - - return self.optimizer.clip_grad_norm( - clip_norm, - aggregate_norm_fn=_aggregate_model_parallel_grad_norm, - ) - - def save_checkpoint(self, filename, extra_state): - """Save all training state in a checkpoint file.""" - extra_state["rng_tracker_states"] = get_cuda_rng_tracker().get_states() - super().save_checkpoint(filename, extra_state) - - def load_checkpoint( - self, - filename, - reset_optimizer=False, - reset_lr_scheduler=False, - optimizer_overrides=None, - reset_meters=False, - ): - extra_state = super().load_checkpoint( - filename, - reset_optimizer=reset_optimizer, - reset_lr_scheduler=reset_lr_scheduler, - optimizer_overrides=optimizer_overrides, - reset_meters=reset_meters, - ) - if extra_state is not None and "rng_tracker_states" in extra_state: - get_cuda_rng_tracker().set_states(extra_state["rng_tracker_states"]) - return extra_state diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/model_parallel/models/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/model_parallel/models/__init__.py deleted file mode 100644 index 4182f6a051a23186d9aea115ed6e28f36b4db378..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/model_parallel/models/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import importlib -import os - -# automatically import any Python files in the models/ directory -models_dir = os.path.dirname(__file__) -for file in os.listdir(models_dir): - path = os.path.join(models_dir, file) - if ( - not file.startswith("_") - and not file.startswith(".") - and (file.endswith(".py") or os.path.isdir(path)) - ): - model_name = file[: file.find(".py")] if file.endswith(".py") else file - module = importlib.import_module("fairseq.model_parallel.models." + model_name) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/model_parallel/models/pipeline_parallel_transformer/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/model_parallel/models/pipeline_parallel_transformer/__init__.py deleted file mode 100644 index 117827c3e9c176477f33e3a6fd7fe19a922411a2..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/model_parallel/models/pipeline_parallel_transformer/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from .model import * # noqa diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/model_parallel/models/pipeline_parallel_transformer/layers.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/model_parallel/models/pipeline_parallel_transformer/layers.py deleted file mode 100644 index 0b9f824cf1b9b64e2cfb2c1dabb34c0b4cb8d873..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/model_parallel/models/pipeline_parallel_transformer/layers.py +++ /dev/null @@ -1,599 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import math -from collections import namedtuple - -import torch -import torch.nn as nn -import torch.nn.functional as F -from fairseq import options, utils -from fairseq.modules import ( - AdaptiveSoftmax, - LayerNorm, - MultiheadAttention, - PositionalEmbedding, -) - -EncoderOut = namedtuple( - "TransformerEncoderOut", - [ - "encoder_out", # T x B x C - "encoder_padding_mask", # B x T - "encoder_embedding", # B x T x C - "encoder_states", # List[T x B x C] - ], -) - - -class TransformerEncoderEmbedding(nn.Module): - """Encoder Embedding + Positional Embedding""" - - def __init__(self, args, embed_tokens): - super().__init__() - self.dropout = args.dropout - self.max_source_positions = args.max_source_positions - self.embed_tokens = embed_tokens - if isinstance(embed_tokens, nn.ModuleList): - self.padding_idx = embed_tokens[0].padding_idx - embed_dim = sum(e.embedding_dim for e in embed_tokens) - else: - self.padding_idx = embed_tokens.padding_idx - embed_dim = embed_tokens.embedding_dim - self.embed_scale = math.sqrt(embed_dim) - self.embed_positions = ( - PositionalEmbedding( - args.max_source_positions, - embed_dim, - self.padding_idx, - learned=args.encoder_learned_pos, - ) - if not args.no_token_positional_embeddings - else None - ) - if getattr(args, "layernorm_embedding", False): - self.layernorm_embedding = LayerNorm(embed_dim) - else: - self.layernorm_embedding = None - - def forward(self, input): - # embed tokens and positions - src_tokens = input[0] - prev_output_tokens = input[2] - if isinstance(self.embed_tokens, nn.ModuleList): - x_embed_list = [] - for embed_tokens_part in self.embed_tokens: - x_embed_list.append(embed_tokens_part(src_tokens)) - - embedded = torch.cat(x_embed_list, dim=-1) - else: - embedded = self.embed_tokens(src_tokens) - x = embed = self.embed_scale * embedded - if self.embed_positions is not None: - x = embed + self.embed_positions(src_tokens) - if self.layernorm_embedding: - x = self.layernorm_embedding(x) - x = F.dropout(x, p=self.dropout, training=self.training) - # B x T x C -> T x B x C - x = x.transpose(0, 1) - - # compute padding mask - encoder_padding_mask = src_tokens.eq(self.padding_idx) - return (x, encoder_padding_mask, prev_output_tokens) - - -class TransformerEncoderLayerNorm(nn.Module): - """ - Layer norm at the the end of all encoder layers if - args.encoder_enormalize_before = True - """ - - def __init__(self, args, embed_dim): - super().__init__() - if args.encoder_normalize_before: - self.layer_norm = LayerNorm(embed_dim) - else: - self.layer_norm = None - - def forward(self, input): - x = input[0] - encoder_padding_mask = input[1] - prev_output_tokens = input[2] - if self.layer_norm: - x = self.layer_norm(x) - # keeping track of the incremental_state is not supported yet - return (x, encoder_padding_mask, prev_output_tokens) - - -class TransformerDecoderEmbedding(nn.Module): - """Decoder Embedding + Positional Embedding""" - - def __init__(self, args, embed_tokens): - super().__init__() - self.dropout = args.dropout - self.share_input_output_embed = args.share_decoder_input_output_embed - input_embed_dim = ( - sum(e.embedding_dim for e in embed_tokens) - if isinstance(embed_tokens, nn.ModuleList) - else embed_tokens.embedding_dim - ) - embed_dim = args.decoder_embed_dim - self.output_embed_dim = args.decoder_output_dim - - padding_idx = ( - embed_tokens[0].padding_idx - if isinstance(embed_tokens, nn.ModuleList) - else embed_tokens.padding_idx - ) - self.max_target_positions = args.max_target_positions - - self.embed_tokens = embed_tokens - self.embed_scale = math.sqrt(embed_dim) # todo: try with input_embed_dim - - self.project_in_dim = ( - Linear(input_embed_dim, embed_dim, bias=False) - if embed_dim != input_embed_dim - else None - ) - - self.embed_positions = ( - PositionalEmbedding( - args.max_target_positions, - embed_dim, - padding_idx, - learned=args.decoder_learned_pos, - ) - if not args.no_token_positional_embeddings - else None - ) - - def forward(self, input): - mt_task = False - if isinstance(input, tuple): - if len(input) == 3: - encoder_out = input[0] - encoder_padding_mask = input[1] - prev_output_tokens = input[2] - incremental_state = None # Hardcoding to avoid passing of None objects - mt_task = True - else: - # HACK for now, need to fix (TODO sidgoyal) - prev_output_tokens = input[0] - # discard "src_lengths" - encoder_out = None - encoder_padding_mask = None - incremental_state = None - - else: - prev_output_tokens = input - encoder_out = None - encoder_padding_mask = None - incremental_state = None - - positions = ( - self.embed_positions( - prev_output_tokens, - incremental_state=incremental_state, - ) - if self.embed_positions is not None - else None - ) - - if incremental_state is not None: - prev_output_tokens = prev_output_tokens[:, -1:] - if positions is not None: - positions = positions[:, -1:] - - # embed tokens and positions - - if isinstance(self.embed_tokens, nn.ModuleList): - x_embed_list = [] - for embed_tokens_part in self.embed_tokens: - x_embed_list.append(embed_tokens_part(prev_output_tokens)) - - x = self.embed_scale * torch.cat(x_embed_list, dim=-1) - else: - x = self.embed_scale * self.embed_tokens(prev_output_tokens) - - if self.project_in_dim is not None: - x = self.project_in_dim(x) - - if positions is not None: - x += positions - x = F.dropout(x, p=self.dropout, training=self.training) - - # B x T x C -> T x B x C - x = x.transpose(0, 1) - if mt_task: - return (x, encoder_out, encoder_padding_mask) - return x - - -class TransformerDecoderOutputLayer(nn.Module): - def __init__(self, args, embed_tokens, dictionary): - super().__init__() - self.share_input_output_embed = args.share_decoder_input_output_embed - self.embed_tokens = embed_tokens - self.output_embed_dim = args.decoder_output_dim - embed_dim = args.decoder_embed_dim - - self.project_out_dim = ( - Linear(embed_dim, self.output_embed_dim, bias=False) - if embed_dim != self.output_embed_dim and not args.tie_adaptive_weights - else None - ) - self.adaptive_softmax = None - if args.adaptive_softmax_cutoff is not None: - assert not isinstance(embed_tokens, nn.ModuleList) - self.adaptive_softmax = AdaptiveSoftmax( - len(dictionary), - self.output_embed_dim, - options.eval_str_list(args.adaptive_softmax_cutoff, type=int), - dropout=args.adaptive_softmax_dropout, - adaptive_inputs=embed_tokens if args.tie_adaptive_weights else None, - factor=args.adaptive_softmax_factor, - tie_proj=args.tie_adaptive_proj, - ) - elif not self.share_input_output_embed: - self.embed_tokens = nn.Parameter( - torch.Tensor(len(dictionary), self.output_embed_dim) - ) - nn.init.normal_( - self.embed_tokens, mean=0, std=self.output_embed_dim**-0.5 - ) - - if args.decoder_normalize_before and not getattr( - args, "no_decoder_final_norm", False - ): - self.layer_norm = LayerNorm(embed_dim) - else: - self.layer_norm = None - - def forward(self, input, apply_final_proj=True): - if isinstance(input, tuple): - x = input[0] - else: - x = input - - if self.layer_norm: - x = self.layer_norm(x) - - # T x B x C -> B x T x C - x = x.transpose(0, 1) - - if self.project_out_dim is not None: - x = self.project_out_dim(x) - if apply_final_proj: - x = self.output_layer(x) - return x - - def output_layer(self, features, **kwargs): - """Project features to the vocabulary size.""" - if self.adaptive_softmax is None: - # project back to size of vocabulary - if self.share_input_output_embed: - if isinstance(self.embed_tokens, nn.ModuleList): - output = None - for i, emb in enumerate(self.embed_tokens): - sidx = i * emb.embedding_dim - eidx = (i + 1) * emb.embedding_dim - if output is None: - output = F.linear(features[:, :, sidx:eidx], emb.weight) - else: - output += F.linear(features[:, :, sidx:eidx], emb.weight) - - return output - else: - return F.linear(features, self.embed_tokens.weight) - else: - return F.linear(features, self.embed_tokens) - else: - return features - - -class TransformerEncoderLayer(nn.Module): - """Encoder layer block. - In the original paper each operation (multi-head attention or FFN) is - postprocessed with: `dropout -> add residual -> layernorm`. In the - tensor2tensor code they suggest that learning is more robust when - preprocessing each layer with layernorm and postprocessing with: - `dropout -> add residual`. We default to the approach in the paper, but the - tensor2tensor approach can be enabled by setting - *args.encoder_normalize_before* to ``True``. - - Args: - args (argparse.Namespace): parsed command-line arguments - """ - - def __init__(self, args): - super().__init__() - self.embed_dim = args.encoder_embed_dim - self.self_attn = MultiheadAttention( - self.embed_dim, - args.encoder_attention_heads, - dropout=args.attention_dropout, - self_attention=True, - ) - self.self_attn_layer_norm = LayerNorm(self.embed_dim) - self.dropout = args.dropout - self.activation_fn = utils.get_activation_fn( - activation=getattr(args, "activation_fn", "relu") - ) - self.activation_dropout = getattr(args, "activation_dropout", 0) - if self.activation_dropout == 0: - # for backwards compatibility with models that use args.relu_dropout - self.activation_dropout = getattr(args, "relu_dropout", 0) - self.normalize_before = args.encoder_normalize_before - self.fc1 = Linear(self.embed_dim, args.encoder_ffn_embed_dim) - self.fc2 = Linear(args.encoder_ffn_embed_dim, self.embed_dim) - self.final_layer_norm = LayerNorm(self.embed_dim) - - def upgrade_state_dict_named(self, state_dict, name): - """ - Rename layer norm states from `...layer_norms.0.weight` to - `...self_attn_layer_norm.weight` and `...layer_norms.1.weight` to - `...final_layer_norm.weight` - """ - layer_norm_map = {"0": "self_attn_layer_norm", "1": "final_layer_norm"} - for old, new in layer_norm_map.items(): - for m in ("weight", "bias"): - k = "{}.layer_norms.{}.{}".format(name, old, m) - if k in state_dict: - state_dict["{}.{}.{}".format(name, new, m)] = state_dict[k] - del state_dict[k] - - def forward(self, input): - """ - Args: - input (Tuple): - input[0] (Tensor): input to the layer of shape `(seq_len, batch, embed_dim)` - input[1] (ByteTensor/FloatTensor): encoder padding mask - - binary ByteTensor of shape `(batch, src_len)` where padding elements - are indicated by ``1``. - input[2] (LongTensor): previous decoder outputs of shape - `(batch, tgt_len)`, for teacher forcing) - Returns: - output (Tuple): - output[0] (Tensor): encoded output of shape `(batch, src_len, embed_dim)` - output[1] (ByteTensor/FloatTensor): encoder padding mask - output[2] (LongTensor): previous decoder outputs - """ - x = input[0] - encoder_padding_mask = input[1] - prev_output_tokens = input[2] - residual = x - x = self.maybe_layer_norm(self.self_attn_layer_norm, x, before=True) - x, _ = self.self_attn( - query=x, key=x, value=x, key_padding_mask=encoder_padding_mask - ) - x = F.dropout(x, p=self.dropout, training=self.training) - x = residual + x - x = self.maybe_layer_norm(self.self_attn_layer_norm, x, after=True) - - residual = x - x = self.maybe_layer_norm(self.final_layer_norm, x, before=True) - x = self.activation_fn(self.fc1(x)) - x = F.dropout(x, p=self.activation_dropout, training=self.training) - x = self.fc2(x) - x = F.dropout(x, p=self.dropout, training=self.training) - x = residual + x - x = self.maybe_layer_norm(self.final_layer_norm, x, after=True) - return (x, encoder_padding_mask, prev_output_tokens) - - def maybe_layer_norm(self, layer_norm, x, before=False, after=False): - assert before ^ after - if after ^ self.normalize_before: - return layer_norm(x) - else: - return x - - -class TransformerDecoderLayer(nn.Module): - """Decoder layer block. - - In the original paper each operation (multi-head attention, encoder - attention or FFN) is postprocessed with: `dropout -> add residual -> - layernorm`. In the tensor2tensor code they suggest that learning is more - robust when preprocessing each layer with layernorm and postprocessing with: - `dropout -> add residual`. We default to the approach in the paper, but the - tensor2tensor approach can be enabled by setting - *args.decoder_normalize_before* to ``True``. - - Args: - args (argparse.Namespace): parsed command-line arguments - no_encoder_attn (bool, optional): whether to attend to encoder outputs - (default: False). - """ - - def __init__( - self, args, no_encoder_attn=False, add_bias_kv=False, add_zero_attn=False - ): - super().__init__() - self.embed_dim = args.decoder_embed_dim - self.self_attn = MultiheadAttention( - embed_dim=self.embed_dim, - num_heads=args.decoder_attention_heads, - dropout=args.attention_dropout, - add_bias_kv=add_bias_kv, - add_zero_attn=add_zero_attn, - self_attention=True, - ) - self.dropout = args.dropout - self.activation_fn = utils.get_activation_fn( - activation=getattr(args, "activation_fn", "relu") - ) - self.activation_dropout = getattr(args, "activation_dropout", 0) - if self.activation_dropout == 0: - # for backwards compatibility with models that use args.relu_dropout - self.activation_dropout = getattr(args, "relu_dropout", 0) - self.normalize_before = args.decoder_normalize_before - - # use layerNorm rather than FusedLayerNorm for exporting. - # char_inputs can be used to determint this. - # TODO remove this once we update apex with the fix - export = getattr(args, "char_inputs", False) - self.self_attn_layer_norm = LayerNorm(self.embed_dim, export=export) - - if no_encoder_attn: - self.encoder_attn = None - self.encoder_attn_layer_norm = None - else: - self.encoder_attn = MultiheadAttention( - self.embed_dim, - args.decoder_attention_heads, - kdim=getattr(args, "encoder_embed_dim", None), - vdim=getattr(args, "encoder_embed_dim", None), - dropout=args.attention_dropout, - encoder_decoder_attention=True, - ) - self.encoder_attn_layer_norm = LayerNorm(self.embed_dim, export=export) - - self.fc1 = Linear(self.embed_dim, args.decoder_ffn_embed_dim) - self.fc2 = Linear(args.decoder_ffn_embed_dim, self.embed_dim) - - self.final_layer_norm = LayerNorm(self.embed_dim, export=export) - self.need_attn = True - - self.onnx_trace = False - - def prepare_for_onnx_export_(self): - self.onnx_trace = True - - def forward(self, input): - """ - Args: - input (Tuple): - input[0] (Tensor): input to the layer of shape `(seq_len, batch, embed_dim)` - input[1] (Tensor): encoder output of shape `(batch, src_len, embed_dim)` - input[2] (ByteTensor/FloatTensor): encoder padding mask - - binary ByteTensor of shape `(batch, src_len)` where padding elements - are indicated by ``1``. - Returns: - output (Tuple): - output[0] (Tensor): encoded output of shape `(batch, src_len, embed_dim)` - output[1] (ByteTensor/FloatTensor): encoder padding mask - output[2] (LongTensor): previous decoder outputs - """ - # Note: incremental state is not yet supported - mt_task = False - if isinstance(input, tuple): - x = input[0] - encoder_out = input[1] - encoder_padding_mask = input[2] - incremental_state = None - mt_task = True - else: - x = input - encoder_out = None - encoder_padding_mask = None - incremental_state = None - - if incremental_state is None: - self_attn_mask = self.buffered_future_mask(x) - else: - self_attn_mask = None - - # TODO: add back prev_self_attn_state, prev_attn_state, - # self_attn_padding_mask - prev_self_attn_state = None - prev_attn_state = None - self_attn_padding_mask = None - - residual = x - x = self.maybe_layer_norm(self.self_attn_layer_norm, x, before=True) - if prev_self_attn_state is not None: - if incremental_state is None: - incremental_state = {} - prev_key, prev_value = prev_self_attn_state - saved_state = {"prev_key": prev_key, "prev_value": prev_value} - self.self_attn._set_input_buffer(incremental_state, saved_state) - x, attn = self.self_attn( - query=x, - key=x, - value=x, - key_padding_mask=self_attn_padding_mask, - incremental_state=incremental_state, - need_weights=False, - attn_mask=self_attn_mask, - ) - x = F.dropout(x, p=self.dropout, training=self.training) - x = residual + x - x = self.maybe_layer_norm(self.self_attn_layer_norm, x, after=True) - - if self.encoder_attn is not None: - residual = x - x = self.maybe_layer_norm(self.encoder_attn_layer_norm, x, before=True) - if prev_attn_state is not None: - if incremental_state is None: - incremental_state = {} - prev_key, prev_value = prev_attn_state - saved_state = {"prev_key": prev_key, "prev_value": prev_value} - self.encoder_attn._set_input_buffer(incremental_state, saved_state) - x, attn = self.encoder_attn( - query=x, - key=encoder_out, - value=encoder_out, - key_padding_mask=encoder_padding_mask, - incremental_state=incremental_state, - static_kv=True, - need_weights=(not self.training and self.need_attn), - ) - x = F.dropout(x, p=self.dropout, training=self.training) - x = residual + x - x = self.maybe_layer_norm(self.encoder_attn_layer_norm, x, after=True) - - residual = x - x = self.maybe_layer_norm(self.final_layer_norm, x, before=True) - x = self.activation_fn(self.fc1(x)) - x = F.dropout(x, p=self.activation_dropout, training=self.training) - x = self.fc2(x) - x = F.dropout(x, p=self.dropout, training=self.training) - x = residual + x - x = self.maybe_layer_norm(self.final_layer_norm, x, after=True) - - if mt_task: - return (x, encoder_out, encoder_padding_mask) - return x - - def buffered_future_mask(self, tensor): - dim = tensor.size(0) - if ( - not hasattr(self, "_future_mask") - or self._future_mask is None - or self._future_mask.device != tensor.device - ): - self._future_mask = torch.triu( - utils.fill_with_neg_inf(tensor.new(dim, dim)), 1 - ) - if self._future_mask.size(0) < dim: - self._future_mask = torch.triu( - utils.fill_with_neg_inf(self._future_mask.resize_(dim, dim)), 1 - ) - return self._future_mask[:dim, :dim] - - def maybe_layer_norm(self, layer_norm, x, before=False, after=False): - assert before ^ after - if after ^ self.normalize_before: - return layer_norm(x) - else: - return x - - def make_generation_fast_(self, need_attn=False, **kwargs): - self.need_attn = need_attn - - -def Embedding(num_embeddings, embedding_dim, padding_idx): - m = nn.Embedding(num_embeddings, embedding_dim, padding_idx=padding_idx) - nn.init.normal_(m.weight, mean=0, std=embedding_dim**-0.5) - nn.init.constant_(m.weight[padding_idx], 0) - return m - - -def Linear(in_features, out_features, bias=True): - m = nn.Linear(in_features, out_features, bias) - nn.init.xavier_uniform_(m.weight) - if bias: - nn.init.constant_(m.bias, 0.0) - return m diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/model_parallel/models/pipeline_parallel_transformer/model.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/model_parallel/models/pipeline_parallel_transformer/model.py deleted file mode 100644 index 3d367b27f9351e374c4c748d02cd0a83208e5af8..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/model_parallel/models/pipeline_parallel_transformer/model.py +++ /dev/null @@ -1,789 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import logging - -import torch -import torch.nn as nn -import torch.nn.functional as F -from fairseq import utils -from fairseq.model_parallel.models.pipeline_parallel_transformer.layers import ( - Embedding, - TransformerDecoderEmbedding, - TransformerDecoderLayer, - TransformerDecoderOutputLayer, - TransformerEncoderEmbedding, - TransformerEncoderLayer, - TransformerEncoderLayerNorm, -) -from fairseq.models import ( - BaseFairseqModel, - FairseqDecoder, - FairseqEncoder, - register_model, - register_model_architecture, -) -from fairseq.models.fairseq_encoder import EncoderOut -from fairseq.models.transformer import ( - base_architecture, - transformer_iwslt_de_en, - transformer_wmt_en_de_big, -) -from fairseq.modules import SinusoidalPositionalEmbedding - -logger = logging.getLogger(__name__) - - -DEFAULT_MAX_SOURCE_POSITIONS = 1024 -DEFAULT_MAX_TARGET_POSITIONS = 1024 -TORCH_PIPE = False -RPC_INIT = False - - -def import_pipe(): - global TORCH_PIPE - global RPC_INIT - try: - from torch.distributed.pipeline.sync import Pipe # noqa - - global Pipe - from torch.distributed.pipeline.sync.utils import partition_model - - global partition_model - import tempfile - - from torch.distributed import rpc - - TORCH_PIPE = True - # Initialize single process RPC agent since TORCH_PIPE requires - # RRef. RRef depends on RPC being initialized and as a result we initialize - # RPC with a single node. - tmpfile = tempfile.NamedTemporaryFile() - if not RPC_INIT: - rpc.init_rpc( - name="worker", - rank=0, - world_size=1, - rpc_backend_options=rpc.TensorPipeRpcBackendOptions( - init_method="file://{}".format(tmpfile.name), - ), - ) - RPC_INIT = True - logger.info("Using torch pipe") - except ImportError: - try: - from fairscale.nn import Pipe # noqa - - logger.info("Using fairscale pipe") - except ImportError: - raise ImportError("Please install fairscale with: pip install fairscale") - - -@register_model("pipeline_parallel_transformer") -class PipelineParallelTransformerModel(BaseFairseqModel): - def __init__(self, encoder, decoder, balance, devices, chunks, checkpoint): - import_pipe() - super().__init__() - assert isinstance(encoder, FairseqEncoder) - assert isinstance(decoder, FairseqDecoder) - encoder_module_list = ( - [encoder.embedding_layer] - + list(encoder.encoder_layers) - + [encoder.final_layer_norm] - ) - self.num_encoder_modules = len(encoder_module_list) - decoder_module_list = ( - [decoder.embedding_layer] - + list(decoder.decoder_layers) - + [decoder.decoder_output_layer] - ) - self.num_decoder_modules = len(decoder_module_list) - module_list = encoder_module_list + decoder_module_list - self.devices = devices - if TORCH_PIPE: - self.model = Pipe( - partition_model(nn.Sequential(*module_list), balance, devices), - chunks=chunks, - checkpoint=checkpoint, - ) - else: - self.model = Pipe( - nn.Sequential(*module_list), - balance=balance, - devices=devices, - chunks=chunks, - checkpoint=checkpoint, - ) - self.encoder_max_positions = self.max_positions_helper( - encoder.embedding_layer, "max_source_positions" - ) - self.decoder_max_positions = self.max_positions_helper( - decoder.embedding_layer, "max_target_positions" - ) - self.adaptive_softmax = getattr(decoder, "adaptive_softmax", None) - # Note: To be populated during inference - self.encoder = None - self.decoder = None - - def forward(self, src_tokens, src_lengths, prev_output_tokens): - if self.training: - input_lst = [src_tokens, src_lengths, prev_output_tokens] - input = tuple(i.to(self.devices[0], non_blocking=True) for i in input_lst) - if TORCH_PIPE: - return self.model(input).local_value() - else: - return self.model(input) - else: - assert self.encoder is not None and self.decoder is not None, ( - "encoder and decoder need to be initialized by " - + "calling the `prepare_for_inference_()` method" - ) - encoder_output_tuple = self.encoder(input) - return self.decoder(encoder_output_tuple) - - def prepare_for_inference_(self, cfg): - if self.encoder is not None and self.decoder is not None: - logger.info("Encoder and Decoder already initialized") - return - encoder_module_list = [] - decoder_module_list = [] - module_count = 0 - for partition in self.model.partitions: - for module in partition: - if module_count < self.num_encoder_modules: - encoder_module_list.append(module) - else: - decoder_module_list.append(module) - module_count += 1 - self.model = None - self.encoder = TransformerEncoder( - cfg.distributed_training, None, None, encoder_module_list - ) - self.decoder = TransformerDecoder( - cfg.distributed_training, - None, - None, - decoder_module_list=decoder_module_list, - ) - - @staticmethod - def add_args(parser): - """Add model-specific arguments to the parser.""" - # fmt: off - parser.add_argument('--activation-fn', - choices=utils.get_available_activation_fns(), - help='activation function to use') - parser.add_argument('--dropout', type=float, metavar='D', - help='dropout probability') - parser.add_argument('--attention-dropout', type=float, metavar='D', - help='dropout probability for attention weights') - parser.add_argument('--activation-dropout', '--relu-dropout', type=float, metavar='D', - help='dropout probability after activation in FFN.') - parser.add_argument('--encoder-embed-path', type=str, metavar='STR', - help='path to pre-trained encoder embedding') - parser.add_argument('--encoder-embed-dim', type=int, metavar='N', - help='encoder embedding dimension') - parser.add_argument('--encoder-ffn-embed-dim', type=int, metavar='N', - help='encoder embedding dimension for FFN') - parser.add_argument('--encoder-layers', type=int, metavar='N', - help='num encoder layers') - parser.add_argument('--encoder-attention-heads', type=int, metavar='N', - help='num encoder attention heads') - parser.add_argument('--encoder-normalize-before', action='store_true', - help='apply layernorm before each encoder block') - parser.add_argument('--encoder-learned-pos', action='store_true', - help='use learned positional embeddings in the encoder') - parser.add_argument('--decoder-embed-path', type=str, metavar='STR', - help='path to pre-trained decoder embedding') - parser.add_argument('--decoder-embed-dim', type=int, metavar='N', - help='decoder embedding dimension') - parser.add_argument('--decoder-ffn-embed-dim', type=int, metavar='N', - help='decoder embedding dimension for FFN') - parser.add_argument('--decoder-layers', type=int, metavar='N', - help='num decoder layers') - parser.add_argument('--decoder-attention-heads', type=int, metavar='N', - help='num decoder attention heads') - parser.add_argument('--decoder-learned-pos', action='store_true', - help='use learned positional embeddings in the decoder') - parser.add_argument('--decoder-normalize-before', action='store_true', - help='apply layernorm before each decoder block') - parser.add_argument('--share-decoder-input-output-embed', action='store_true', - help='share decoder input and output embeddings') - parser.add_argument('--share-all-embeddings', action='store_true', - help='share encoder, decoder and output embeddings' - ' (requires shared dictionary and embed dim)') - parser.add_argument('--no-token-positional-embeddings', default=False, action='store_true', - help='if set, disables positional embeddings (outside self attention)') - parser.add_argument('--adaptive-softmax-cutoff', metavar='EXPR', - help='comma separated list of adaptive softmax cutoff points. ' - 'Must be used with adaptive_loss criterion'), - parser.add_argument('--adaptive-softmax-dropout', type=float, metavar='D', - help='sets adaptive softmax dropout for the tail projections') - parser.add_argument('--num-embedding-chunks', type=int, metavar='N', default=1, - help='Number of embedding layer chunks (enables more even distribution' - 'of optimizer states across data parallel nodes' - 'when using optimizer state sharding and' - 'a big embedding vocabulary)') - # fmt: on - - @classmethod - def build_model_base(cls, args, task): - """Build a new model instance.""" - - # make sure all arguments are present in older models - base_architecture(args) - - if not hasattr(args, "max_source_positions"): - args.max_source_positions = DEFAULT_MAX_SOURCE_POSITIONS - if not hasattr(args, "max_target_positions"): - args.max_target_positions = DEFAULT_MAX_TARGET_POSITIONS - - src_dict, tgt_dict = task.source_dictionary, task.target_dictionary - - def build_embedding(dictionary, embed_dim, path=None, num_embed_chunks=1): - assert embed_dim % num_embed_chunks == 0, ( - f"Number of embedding chunks = {num_embed_chunks} should be " - + f"divisible by the embedding dimension = {embed_dim}" - ) - assert path is None or num_embed_chunks == 1, ( - "Loading embedding from a path with number of embedding chunks > 1" - + " is not yet supported" - ) - num_embeddings = len(dictionary) - padding_idx = dictionary.pad() - # if provided, load from preloaded dictionaries - if path: - emb = Embedding(num_embeddings, embed_dim, padding_idx) - embed_dict = utils.parse_embedding(path) - utils.load_embedding(embed_dict, dictionary, emb) - else: - embed_chunk_dim = embed_dim // num_embed_chunks - emb = nn.ModuleList() - for i in range(num_embed_chunks): - emb.append(Embedding(num_embeddings, embed_chunk_dim, padding_idx)) - return emb - - num_embed_chunks = args.num_embedding_chunks - if args.share_all_embeddings: - if src_dict != tgt_dict: - raise ValueError("--share-all-embeddings requires a joined dictionary") - if args.encoder_embed_dim != args.decoder_embed_dim: - raise ValueError( - "--share-all-embeddings requires --encoder-embed-dim to match --decoder-embed-dim" - ) - if args.decoder_embed_path and ( - args.decoder_embed_path != args.encoder_embed_path - ): - raise ValueError( - "--share-all-embeddings not compatible with --decoder-embed-path" - ) - encoder_embed_tokens = build_embedding( - src_dict, - args.encoder_embed_dim, - args.encoder_embed_path, - num_embed_chunks, - ) - decoder_embed_tokens = encoder_embed_tokens - args.share_decoder_input_output_embed = True - else: - assert args.share_decoder_input_output_embed or num_embed_chunks == 1, ( - "Not sharing decoder I/O embeddings is not yet supported with number of " - + "embedding chunks > 1" - ) - encoder_embed_tokens = build_embedding( - src_dict, - args.encoder_embed_dim, - args.encoder_embed_path, - num_embed_chunks, - ) - decoder_embed_tokens = build_embedding( - tgt_dict, - args.decoder_embed_dim, - args.decoder_embed_path, - num_embed_chunks, - ) - - encoder = cls.build_encoder(args, src_dict, encoder_embed_tokens) - decoder = cls.build_decoder(args, tgt_dict, decoder_embed_tokens) - return (encoder, decoder) - - @classmethod - def build_encoder(cls, args, src_dict, embed_tokens): - return TransformerEncoder(args, src_dict, embed_tokens) - - @classmethod - def build_decoder(cls, args, tgt_dict, embed_tokens): - return TransformerDecoder(args, tgt_dict, embed_tokens) - - @classmethod - def build_model(cls, args, task): - encoder, decoder = cls.build_model_base(args, task) - return PipelineParallelTransformerModel( - encoder=encoder, - decoder=decoder, - balance=utils.eval_str_list(args.pipeline_balance, type=int), - devices=utils.eval_str_list(args.pipeline_devices, type=int), - chunks=args.pipeline_chunks, - checkpoint=args.pipeline_checkpoint, - ) - - def output_layer(self, features, **kwargs): - """Project features to the default output size (typically vocabulary size).""" - return self.decoder.output_layer(features, **kwargs) - - def max_positions(self): - """Maximum length supported by the model.""" - return (self.encoder_max_positions, self.decoder_max_positions) - - def max_positions_helper( - self, embedding_layer, max_positions_field="max_source_positions" - ): - """Maximum input length supported by the encoder or decoder.""" - if embedding_layer.embed_positions is None: - return getattr(embedding_layer, max_positions_field) - return min( - getattr(embedding_layer, max_positions_field), - embedding_layer.embed_positions.max_positions, - ) - - def get_normalized_probs(self, net_output, log_probs, sample=None): - """Get normalized probabilities (or log probs) from a net's output.""" - - if hasattr(self, "adaptive_softmax") and self.adaptive_softmax is not None: - if sample is not None: - assert "target" in sample - target = sample["target"] - else: - target = None - out = self.adaptive_softmax.get_log_prob(net_output, target=target) - return out.exp_() if not log_probs else out - - # A Pipe() module returns a tuple of tensors as the output. - # In this case, the tuple has one element - the output tensor of logits - logits = net_output if isinstance(net_output, torch.Tensor) else net_output[0] - if log_probs: - return utils.log_softmax(logits, dim=-1, onnx_trace=False) - else: - return utils.softmax(logits, dim=-1, onnx_trace=False) - - def max_decoder_positions(self): - """Maximum length supported by the decoder.""" - return self.decoder_max_positions - - def load_state_dict(self, state_dict, strict=True, model_cfg=None): - """Copies parameters and buffers from *state_dict* into this module and - its descendants. - - Overrides the method in :class:`nn.Module`. Compared with that method - this additionally "upgrades" *state_dicts* from old checkpoints. - """ - self.upgrade_state_dict(state_dict) - is_regular_transformer = not any("model.partitions" in k for k in state_dict) - if is_regular_transformer: - state_dict = self.convert_to_pipeline_parallel_state_dict(state_dict) - return super().load_state_dict(state_dict, strict) - - def convert_to_pipeline_parallel_state_dict(self, state_dict): - new_state_dict = self.state_dict() - encoder_layer_idx = 0 - decoder_layer_idx = 0 - encoder_key_suffixes = [ - "self_attn.k_proj.weight", - "self_attn.k_proj.bias", - "self_attn.v_proj.weight", - "self_attn.v_proj.bias", - "self_attn.q_proj.weight", - "self_attn.q_proj.bias", - "self_attn.out_proj.weight", - "self_attn.out_proj.bias", - "self_attn_layer_norm.weight", - "self_attn_layer_norm.bias", - "fc1.weight", - "fc1.bias", - "fc2.weight", - "fc2.bias", - "final_layer_norm.weight", - "final_layer_norm.bias", - ] - decoder_key_suffixes = [ - "self_attn.k_proj.weight", - "self_attn.k_proj.bias", - "self_attn.v_proj.weight", - "self_attn.v_proj.bias", - "self_attn.q_proj.weight", - "self_attn.q_proj.bias", - "self_attn.out_proj.weight", - "self_attn.out_proj.bias", - "self_attn_layer_norm.weight", - "self_attn_layer_norm.bias", - "encoder_attn.k_proj.weight", - "encoder_attn.k_proj.bias", - "encoder_attn.v_proj.weight", - "encoder_attn.v_proj.bias", - "encoder_attn.q_proj.weight", - "encoder_attn.q_proj.bias", - "encoder_attn.out_proj.weight", - "encoder_attn.out_proj.bias", - "encoder_attn_layer_norm.weight", - "encoder_attn_layer_norm.bias", - "fc1.weight", - "fc1.bias", - "fc2.weight", - "fc2.bias", - "final_layer_norm.weight", - "final_layer_norm.bias", - ] - for pid, partition in enumerate(self.model.partitions): - logger.info(f"Begin Partition {pid}") - for mid, module in enumerate(partition): - # fmt: off - if isinstance(module, TransformerEncoderEmbedding): - new_state_dict[f'model.partitions.{pid}.{mid}.embed_tokens.weight'] = state_dict['encoder.embed_tokens.weight'] - new_state_dict[f'model.partitions.{pid}.{mid}.embed_positions._float_tensor'] = state_dict['encoder.embed_positions._float_tensor'] - if isinstance(module, TransformerEncoderLayer): - for suffix in encoder_key_suffixes: - new_state_dict[f'model.partitions.{pid}.{mid}.{suffix}'] = state_dict[f'encoder.layers.{encoder_layer_idx}.{suffix}'] - encoder_layer_idx += 1 - if isinstance(module, TransformerDecoderLayer): - for suffix in decoder_key_suffixes: - new_state_dict[f'model.partitions.{pid}.{mid}.{suffix}'] = state_dict[f'decoder.layers.{decoder_layer_idx}.{suffix}'] - decoder_layer_idx += 1 - if isinstance(module, TransformerEncoderLayerNorm): - if 'encoder.layer_norm.weight' in state_dict: - new_state_dict[f'model.partitions.{pid}.{mid}.layer_norm.weight'] = state_dict['encoder.layer_norm.weight'] - new_state_dict[f'model.partitions.{pid}.{mid}.layer_norm.bias'] = state_dict['encoder.layer_norm.bias'] - if isinstance(module, TransformerDecoderEmbedding): - new_state_dict[f'model.partitions.{pid}.{mid}.embed_tokens.weight'] = state_dict['decoder.embed_tokens.weight'] - new_state_dict[f'model.partitions.{pid}.{mid}.embed_positions._float_tensor'] = state_dict['decoder.embed_positions._float_tensor'] - if isinstance(module, TransformerDecoderOutputLayer): - new_state_dict[f'model.partitions.{pid}.{mid}.output_projection.weight'] = state_dict['decoder.output_projection.weight'] - # fmt: on - return new_state_dict - - -class TransformerEncoder(FairseqEncoder): - """ - Transformer encoder consisting of *args.encoder_layers* layers. Each layer - is a :class:`TransformerEncoderLayer`. - - Args: - args (argparse.Namespace): parsed command-line arguments - dictionary (~fairseq.data.Dictionary): encoding dictionary - embed_tokens (torch.nn.Embedding): input embedding - """ - - def __init__(self, args, dictionary, embed_tokens, encoder_module_list=None): - super().__init__(dictionary) - self.register_buffer("version", torch.Tensor([3])) - import_pipe() - self.use_pipeline = encoder_module_list is not None - if not self.use_pipeline: - self.embedding_layer = TransformerEncoderEmbedding(args, embed_tokens) - self.encoder_layers = nn.Sequential( - *[TransformerEncoderLayer(args) for i in range(args.encoder_layers)] - ) - if isinstance(embed_tokens, nn.ModuleList): - emb_dim = sum(e.embedding_dim for e in embed_tokens) - else: - emb_dim = embed_tokens.embedding_dim - self.final_layer_norm = TransformerEncoderLayerNorm(args, emb_dim) - else: - encoder_balance = utils.eval_str_list( - args.pipeline_encoder_balance, type=int - ) - encoder_devices = utils.eval_str_list( - args.pipeline_encoder_devices, type=int - ) - assert sum(encoder_balance) == len(encoder_module_list), ( - f"Sum of encoder_balance={encoder_balance} is not equal " - + f"to num_encoder_modules={len(encoder_module_list)}" - ) - if TORCH_PIPE: - self.model = Pipe( - module=partition_model( - nn.Sequential(*encoder_module_list), - encoder_balance, - encoder_devices, - ), - chunks=args.pipeline_chunks, - checkpoint=args.pipeline_checkpoint, - ) - else: - self.model = Pipe( - module=nn.Sequential(*encoder_module_list), - balance=encoder_balance, - devices=encoder_devices, - chunks=args.pipeline_chunks, - checkpoint=args.pipeline_checkpoint, - ) - - def forward(self, src_tokens, src_lengths): - """ - Args: - input_tuple( - src_tokens (LongTensor): tokens in the source language of shape - `(batch, src_len)` - src_lengths (torch.LongTensor): lengths of each source sentence of - shape `(batch)` - ) - - Returns: - output_tuple( - - **encoder_out** (Tensor): the last encoder layer's output of - shape `(src_len, batch, embed_dim)` - - **encoder_padding_mask** (ByteTensor): the positions of - padding elements of shape `(batch, src_len)` - - prev_output_tokens - - **encoder_states** (List[Tensor]): all intermediate - hidden states of shape `(src_len, batch, embed_dim)`. - Only populated if *return_all_hiddens* is True. - ) - """ - dummy_prev_output_tokens = torch.zeros( - 1, dtype=src_tokens.dtype, device=src_tokens.device - ) - input_tuple = (src_tokens, src_lengths, dummy_prev_output_tokens) - if self.use_pipeline: - input_tuple = tuple(i.to(self.model.devices[0]) for i in input_tuple) - if TORCH_PIPE: - encoder_out = self.model(input_tuple).local_value() - else: - encoder_out = self.model(input_tuple) - else: - encoder_embed_output_tuple = self.embedding_layer(input_tuple) - encoder_layers_output = self.encoder_layers(encoder_embed_output_tuple) - encoder_out = self.final_layer_norm(encoder_layers_output) - # first element is the encoder output - # second element is the encoder padding mask - # the remaining elements of EncoderOut are not computed by - # the PipelineParallelTransformer - return EncoderOut(encoder_out[0], encoder_out[1], None, None, None, None) - - def reorder_encoder_out(self, encoder_out, new_order): - """ - Reorder encoder output according to *new_order*. - - Args: - encoder_out: output from the ``forward()`` method - new_order (LongTensor): desired order - - Returns: - *encoder_out* rearranged according to *new_order* - """ - if encoder_out.encoder_out is not None: - encoder_out = encoder_out._replace( - encoder_out=encoder_out.encoder_out.index_select(1, new_order) - ) - if encoder_out.encoder_padding_mask is not None: - encoder_out = encoder_out._replace( - encoder_padding_mask=encoder_out.encoder_padding_mask.index_select( - 0, new_order - ) - ) - if encoder_out.encoder_embedding is not None: - encoder_out = encoder_out._replace( - encoder_embedding=encoder_out.encoder_embedding.index_select( - 0, new_order - ) - ) - if encoder_out.encoder_states is not None: - for idx, state in enumerate(encoder_out.encoder_states): - encoder_out.encoder_states[idx] = state.index_select(1, new_order) - return encoder_out - - def max_positions(self): - """Maximum input length supported by the encoder.""" - if self.embedding_layer.embed_positions is None: - return self.embedding_layer.max_source_positions - return min( - self.embedding_layer.max_source_positions, - self.embedding_layer.embed_positions.max_positions, - ) - - -class TransformerDecoder(FairseqDecoder): - """ - Transformer decoder consisting of *args.decoder_layers* layers. Each layer - is a :class:`TransformerDecoderLayer`. - - Args: - args (argparse.Namespace): parsed command-line arguments - dictionary (~fairseq.data.Dictionary): decoding dictionary - embed_tokens (torch.nn.Embedding): output embedding - no_encoder_attn (bool, optional): whether to attend to encoder outputs - (default: False). - """ - - def __init__( - self, - args, - dictionary, - embed_tokens, - no_encoder_attn=False, - decoder_module_list=None, - ): - super().__init__(dictionary) - self.register_buffer("version", torch.Tensor([3])) - import_pipe() - self.use_pipeline = decoder_module_list is not None - if not self.use_pipeline: - self.embedding_layer = TransformerDecoderEmbedding(args, embed_tokens) - self.decoder_layers = nn.Sequential( - *[ - TransformerDecoderLayer(args, no_encoder_attn) - for _ in range(args.decoder_layers) - ] - ) - self.decoder_output_layer = TransformerDecoderOutputLayer( - args, embed_tokens, dictionary - ) - else: - decoder_balance = utils.eval_str_list( - args.pipeline_decoder_balance, type=int - ) - decoder_devices = utils.eval_str_list( - args.pipeline_decoder_devices, type=int - ) - assert sum(decoder_balance) == len(decoder_module_list), ( - f"Sum of decoder_balance={decoder_balance} is not equal " - + f"to num_decoder_modules={len(decoder_module_list)}" - ) - if TORCH_PIPE: - self.model = Pipe( - module=partition_model( - nn.Sequential(*decoder_module_list), - decoder_balance, - decoder_devices, - ), - chunks=args.pipeline_chunks, - checkpoint=args.pipeline_checkpoint, - ) - else: - self.model = Pipe( - module=nn.Sequential(*decoder_module_list), - balance=decoder_balance, - devices=decoder_devices, - chunks=args.pipeline_chunks, - checkpoint=args.pipeline_checkpoint, - ) - - def forward( - self, - prev_output_tokens, - encoder_out=None, - ): - """ - Args: - prev_output_tokens (LongTensor): previous decoder outputs of shape - `(batch, tgt_len)`, for teacher forcing - encoder_out (optional): output from the encoder, used for - encoder-side attention - incremental_state (dict): dictionary used for storing state during - :ref:`Incremental decoding` - features_only (bool, optional): only return features without - applying output layer (default: False). - - Returns: - tuple: - - the decoder's output of shape `(batch, tgt_len, vocab)` - - a dictionary with any model-specific outputs - """ - input_tuple = ( - encoder_out.encoder_out, - encoder_out.encoder_padding_mask, - prev_output_tokens, - ) - if self.use_pipeline: - input_tuple = tuple(i.to(self.model.devices[0]) for i in input_tuple) - if TORCH_PIPE: - return (self.model(input_tuple).local_value(),) - else: - return (self.model(input_tuple),) - else: - embed_layer_output = self.embedding_layer(input_tuple) - state = self.decoder_layers(embed_layer_output) - return (self.decoder_output_layer(state),) - - def output_layer(self, features, **kwargs): - """Project features to the vocabulary size.""" - if self.adaptive_softmax is None: - # project back to size of vocabulary - if self.share_input_output_embed: - return F.linear(features, self.embed_tokens.weight) - else: - return F.linear(features, self.embed_out) - else: - return features - - def max_positions(self): - """Maximum output length supported by the decoder.""" - if self.embedding_layer.embed_positions is None: - return self.embedding_layer.max_target_positions - return min( - self.embedding_layer.max_target_positions, - self.embedding_layer.embed_positions.max_positions, - ) - - def buffered_future_mask(self, tensor): - dim = tensor.size(0) - if ( - not hasattr(self, "_future_mask") - or self._future_mask is None - or self._future_mask.device != tensor.device - or self._future_mask.size(0) < dim - ): - self._future_mask = torch.triu( - utils.fill_with_neg_inf(tensor.new(dim, dim)), 1 - ) - return self._future_mask[:dim, :dim] - - def upgrade_state_dict_named(self, state_dict, name): - """Upgrade a (possibly old) state dict for new versions of fairseq.""" - if isinstance(self.embed_positions, SinusoidalPositionalEmbedding): - weights_key = "{}.embed_positions.weights".format(name) - if weights_key in state_dict: - del state_dict[weights_key] - state_dict[ - "{}.embed_positions._float_tensor".format(name) - ] = torch.FloatTensor(1) - - for i in range(len(self.layers)): - # update layer norms - layer_norm_map = { - "0": "self_attn_layer_norm", - "1": "encoder_attn_layer_norm", - "2": "final_layer_norm", - } - for old, new in layer_norm_map.items(): - for m in ("weight", "bias"): - k = "{}.layers.{}.layer_norms.{}.{}".format(name, i, old, m) - if k in state_dict: - state_dict[ - "{}.layers.{}.{}.{}".format(name, i, new, m) - ] = state_dict[k] - del state_dict[k] - - version_key = "{}.version".format(name) - if utils.item(state_dict.get(version_key, torch.Tensor([1]))[0]) <= 2: - # earlier checkpoints did not normalize after the stack of layers - self.layer_norm = None - self.normalize = False - state_dict[version_key] = torch.Tensor([1]) - - return state_dict - - -@register_model_architecture( - "pipeline_parallel_transformer", "transformer_iwslt_de_en_pipeline_parallel" -) -def transformer_iwslt_de_en_dist(args): - transformer_iwslt_de_en(args) - - -@register_model_architecture( - "pipeline_parallel_transformer", "transformer_wmt_en_de_big_pipeline_parallel" -) -def transformer_wmt_en_de_big_dist(args): - transformer_wmt_en_de_big(args) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/model_parallel/models/roberta/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/model_parallel/models/roberta/__init__.py deleted file mode 100644 index 117827c3e9c176477f33e3a6fd7fe19a922411a2..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/model_parallel/models/roberta/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from .model import * # noqa diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/model_parallel/models/roberta/model.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/model_parallel/models/roberta/model.py deleted file mode 100644 index fe98ccb740e46dd96590b93855af9865f36fbae5..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/model_parallel/models/roberta/model.py +++ /dev/null @@ -1,224 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. -""" -RoBERTa: A Robustly Optimized BERT Pretraining Approach. -""" - -import logging - -import torch -import torch.nn as nn -import torch.nn.functional as F -from fairseq import utils -from fairseq.model_parallel.models.transformer import ModelParallelTransformerEncoder -from fairseq.models import register_model, register_model_architecture -from fairseq.models.roberta import ( - RobertaEncoder, - RobertaModel, - roberta_base_architecture, - roberta_prenorm_architecture, -) -from fairseq.modules import LayerNorm - -try: - from fairseq.model_parallel.megatron.mpu import ( - ColumnParallelLinear, - VocabParallelEmbedding, - copy_to_model_parallel_region, - gather_from_model_parallel_region, - ) - - has_megatron_submodule = True -except (ImportError, ModuleNotFoundError): - has_megatron_submodule = False - -logger = logging.getLogger(__name__) - - -@register_model("model_parallel_roberta") -class ModelParallelRobertaModel(RobertaModel): - def __init__(self, args, encoder): - super().__init__(args, encoder) - - self.classification_heads = nn.ModuleDict() - - @staticmethod - def add_args(parser): - RobertaModel.add_args(parser) - parser.add_argument( - "--no-final-layer-norm", - action="store_true", - help=( - "don't add final layernorm (only applicable when " - "--encoder-normalize-before=True" - ), - ) - - @classmethod - def build_model(cls, args, task): - """Build a new model instance.""" - - # make sure all arguments are present - base_architecture(args) - - task.source_dictionary.pad_to_multiple_(args.model_parallel_size * 8) - task.target_dictionary.pad_to_multiple_(args.model_parallel_size * 8) - - if not hasattr(args, "max_positions"): - args.max_positions = args.tokens_per_sample - - if getattr(args, "untie_weights_roberta", False): - raise NotImplementedError( - "--untie-weights-roberta is not supported in model parallel mode" - ) - - encoder = ModelParallelRobertaEncoder(args, task.source_dictionary) - return cls(args, encoder) - - def forward( - self, - src_tokens, - features_only=False, - return_all_hiddens=False, - classification_head_name=None, - **kwargs - ): - if classification_head_name is not None: - features_only = True - - x, extra = self.encoder(src_tokens, features_only, return_all_hiddens, **kwargs) - - if classification_head_name is not None: - x = self.classification_heads[classification_head_name](x) - return x, extra - - def register_classification_head( - self, name, num_classes=None, inner_dim=None, **kwargs - ): - """Register a classification head.""" - if name in self.classification_heads: - prev_num_classes = self.classification_heads[name].out_proj.out_features - prev_inner_dim = self.classification_heads[name].dense.out_features - if num_classes != prev_num_classes or inner_dim != prev_inner_dim: - logger.warning( - 're-registering head "{}" with num_classes {} (prev: {}) ' - "and inner_dim {} (prev: {})".format( - name, num_classes, prev_num_classes, inner_dim, prev_inner_dim - ) - ) - self.classification_heads[name] = ModelParallelRobertaClassificationHead( - self.args.encoder_embed_dim, - inner_dim or self.args.encoder_embed_dim, - num_classes, - self.args.pooler_activation_fn, - self.args.pooler_dropout, - ) - - -class ModelParallelRobertaLMHead(nn.Module): - """Head for masked language modeling.""" - - def __init__(self, embed_dim, output_dim, activation_fn, weight=None): - super().__init__() - self.dense = ColumnParallelLinear(embed_dim, embed_dim, gather_output=True) - self.activation_fn = utils.get_activation_fn(activation_fn) - self.layer_norm = LayerNorm(embed_dim) - - if weight is None: - weight = nn.Linear(embed_dim, output_dim, bias=False).weight - self.weight = weight - self.bias = nn.Parameter(torch.zeros(output_dim)) - - def forward(self, features, masked_tokens=None, **kwargs): - # Only project the unmasked tokens while training, - # saves both memory and computation - if masked_tokens is not None: - features = features[masked_tokens, :] - - x = self.dense(features) - x = self.activation_fn(x) - x = self.layer_norm(x) - - x = copy_to_model_parallel_region(x) - # project back to size of vocabulary with bias - x = F.linear(x, self.weight) - x = gather_from_model_parallel_region(x).contiguous() - x = x + self.bias - return x - - -class ModelParallelRobertaClassificationHead(nn.Module): - """Head for sentence-level classification tasks.""" - - def __init__( - self, input_dim, inner_dim, num_classes, activation_fn, pooler_dropout - ): - super().__init__() - self.dense = ColumnParallelLinear(input_dim, inner_dim, gather_output=True) - self.activation_fn = utils.get_activation_fn(activation_fn) - self.dropout = nn.Dropout(p=pooler_dropout) - self.out_proj = nn.Linear(inner_dim, num_classes) - - def forward(self, features, **kwargs): - x = features[:, 0, :] # take <s> token (equiv. to [CLS]) - x = self.dropout(x) - x = self.dense(x) - x = self.activation_fn(x) - x = self.dropout(x) - x = self.out_proj(x) - return x - - -class ModelParallelRobertaEncoder(RobertaEncoder): - """RoBERTa encoder.""" - - def __init__(self, args, dictionary): - super().__init__(args, dictionary) - assert not self.args.untie_weights_roberta - - def build_embedding(self, vocab_size, embedding_dim, padding_idx): - return VocabParallelEmbedding(vocab_size, embedding_dim, padding_idx) - - def build_encoder(self, args, dictionary, embed_tokens): - return ModelParallelTransformerEncoder(args, dictionary, embed_tokens) - - def build_lm_head(self, embed_dim, output_dim, activation_fn, weight): - return ModelParallelRobertaLMHead(embed_dim, output_dim, activation_fn, weight) - - -@register_model_architecture("model_parallel_roberta", "model_parallel_roberta") -def base_architecture(args): - args.no_final_layer_norm = getattr(args, "no_final_layer_norm", False) - # model parallel RoBERTa defaults to "Pre-LN" formulation - roberta_prenorm_architecture(args) - - -# earlier versions of model parallel RoBERTa removed the final layer norm -@register_model_architecture("model_parallel_roberta", "model_parallel_roberta_v1") -def model_parallel_roberta_v1_architecture(args): - args.no_final_layer_norm = getattr(args, "no_final_layer_norm", True) - base_architecture(args) - - -@register_model_architecture( - "model_parallel_roberta", "model_parallel_roberta_postnorm" -) -def model_parallel_roberta_postnorm_architecture(args): - # the original BERT/RoBERTa uses the "Post-LN" formulation - roberta_base_architecture(args) - - -@register_model_architecture("model_parallel_roberta", "model_parallel_roberta_base") -def model_parallel_roberta_base_architecture(args): - base_architecture(args) - - -@register_model_architecture("model_parallel_roberta", "model_parallel_roberta_large") -def model_parallel_roberta_large_architecture(args): - args.encoder_layers = getattr(args, "encoder_layers", 24) - args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 1024) - args.encoder_ffn_embed_dim = getattr(args, "encoder_ffn_embed_dim", 4096) - args.encoder_attention_heads = getattr(args, "encoder_attention_heads", 16) - base_architecture(args) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/model_parallel/models/transformer.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/model_parallel/models/transformer.py deleted file mode 100644 index 9d4492d1e9d96fe7d2dc75c4cc383456890f1bcd..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/model_parallel/models/transformer.py +++ /dev/null @@ -1,120 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import logging - -import torch.nn as nn -from fairseq.model_parallel.modules import ( - ModelParallelTransformerDecoderLayer, - ModelParallelTransformerEncoderLayer, -) -from fairseq.models import register_model -from fairseq.models.transformer import ( - TransformerDecoder, - TransformerEncoder, - TransformerModel, -) - -try: - from fairseq.model_parallel.megatron.mpu import ( - VocabParallelEmbedding, - copy_to_model_parallel_region, - gather_from_model_parallel_region, - ) - - has_megatron_submodule = True -except (ImportError, ModuleNotFoundError): - has_megatron_submodule = False - - -logger = logging.getLogger(__name__) - - -@register_model("model_parallel_transformer") -class ModelParallelTransformerModel(TransformerModel): - """ - Model parallel Transformer model. - """ - - @classmethod - def build_embedding(cls, args, dictionary, embed_dim, path=None): - if not has_megatron_submodule: - raise ImportError( - "\n\nPlease install the megatron submodule:" - "\n\n git submodule update --init " - "fairseq/model_parallel/megatron" - ) - dictionary.pad_to_multiple_(args.model_parallel_size * 8) - num_embeddings = len(dictionary) - padding_idx = dictionary.pad() - - def _vocab_init(tensor, **kwargs): - nn.init.normal_(tensor, mean=0, std=num_embeddings**-0.5) - nn.init.constant_(tensor[1], 0) - - emb = VocabParallelEmbedding( - num_embeddings, embed_dim, padding_idx, init_method=_vocab_init - ) - # if provided, load from preloaded dictionaries - if path: - raise NotImplementedError( - "Loading of embedding from path is not supported for model parallel" - ) - return emb - - @classmethod - def build_encoder(cls, args, src_dict, embed_tokens): - return ModelParallelTransformerEncoder(args, src_dict, embed_tokens) - - @classmethod - def build_decoder(cls, args, tgt_dict, embed_tokens): - return ModelParallelTransformerDecoder( - args, - tgt_dict, - embed_tokens, - no_encoder_attn=getattr(args, "no_cross_attention", False), - ) - - -class ModelParallelTransformerEncoder(TransformerEncoder): - """ - Model parallel Transformer encoder consisting of *args.encoder_layers* layers. Each layer - is a :class:`ModelParallelTransformerEncoderLayer`. - """ - - def __init__(self, args, dictionary, embed_tokens): - super().__init__(args, dictionary, embed_tokens) - - if args.no_final_layer_norm: - self.layer_norm = None - - def build_encoder_layer(self, args): - return ModelParallelTransformerEncoderLayer(args) - - -class ModelParallelTransformerDecoder(TransformerDecoder): - """ - Model Parallel Transformer decoder consisting of *args.decoder_layers* layers. Each layer - is a :class:`ModelParallelTransformerDecoderLayer`. - """ - - def build_decoder_layer(self, args, no_encoder_attn=False): - return ModelParallelTransformerDecoderLayer(args, no_encoder_attn) - - def output_layer(self, features, **kwargs): - """Project features to the vocabulary size.""" - if not self.share_input_output_embed: - raise NotImplementedError( - "Model parallel training currently requires --share-decoder-input-output-embed" - ) - - features = copy_to_model_parallel_region(features) - - # project back to size of vocabulary - x = self.output_projection(features) - - if getattr(self.args, "criterion") != "vocab_parallel_cross_entropy": - x = gather_from_model_parallel_region(x).contiguous() - return x diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/model_parallel/models/transformer_lm.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/model_parallel/models/transformer_lm.py deleted file mode 100644 index 496c3ea858449a74d8cb881d6f64575ea92b6029..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/model_parallel/models/transformer_lm.py +++ /dev/null @@ -1,168 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import torch.nn as nn -from fairseq.model_parallel.models.transformer import ModelParallelTransformerDecoder -from fairseq.models import register_model, register_model_architecture -from fairseq.models.transformer_lm import TransformerLanguageModel - -try: - from fairseq.model_parallel.megatron.mpu import VocabParallelEmbedding - - has_megatron_submodule = True -except (ImportError, ModuleNotFoundError): - has_megatron_submodule = False - - -DEFAULT_MAX_TARGET_POSITIONS = 1024 - - -@register_model("model_parallel_transformer_lm") -class ModelParallelTransformerLanguageModel(TransformerLanguageModel): - @staticmethod - def add_args(parser): - TransformerLanguageModel.add_args(parser) - - @classmethod - def build_model(cls, args, task): - """Build a new model instance.""" - if not has_megatron_submodule: - raise ImportError( - "\n\nPlease install the megatron submodule:" - "\n\n git submodule update --init " - "fairseq/model_parallel/megatron" - ) - - # make sure all arguments are present in older models - base_lm_architecture(args) - - task.source_dictionary.pad_to_multiple_(args.model_parallel_size * 8) - task.target_dictionary.pad_to_multiple_(args.model_parallel_size * 8) - - if args.decoder_layers_to_keep: - args.decoder_layers = len(args.decoder_layers_to_keep.split(",")) - - if getattr(args, "max_target_positions", None) is None: - args.max_target_positions = getattr( - args, "tokens_per_sample", DEFAULT_MAX_TARGET_POSITIONS - ) - - if args.character_embeddings: - raise NotImplementedError( - "Character embeddings is not supported for model parallel" - ) - elif args.adaptive_input: - raise NotImplementedError( - "Adaptive input is not supported for model parallel" - ) - else: - embed_tokens = cls.build_embedding( - args, task.source_dictionary, args.decoder_input_dim - ) - - decoder = ModelParallelTransformerDecoder( - args, - task.target_dictionary, - embed_tokens, - no_encoder_attn=True, - ) - return cls(decoder) - - @classmethod - def build_embedding(cls, args, dictionary, embed_dim, path=None): - def _vocab_init(tensor, **kwargs): - nn.init.normal_(tensor, mean=0, std=embed_dim**-0.5) - nn.init.constant_(tensor[1], 0) - - embed_tokens = VocabParallelEmbedding( - len(dictionary), embed_dim, dictionary.pad(), init_method=_vocab_init - ) - return embed_tokens - - -def base_lm_architecture(args): - # backward compatibility for older model checkpoints - if hasattr(args, "no_tie_adaptive_proj"): - # previous models defined --no-tie-adaptive-proj, so use the existence of - # that option to determine if this is an "old" model checkpoint - args.no_decoder_final_norm = True # old models always set this to True - if args.no_tie_adaptive_proj is False: - args.tie_adaptive_proj = True - if hasattr(args, "decoder_final_norm"): - args.no_decoder_final_norm = not args.decoder_final_norm - - args.activation_fn = getattr(args, "activation_fn", "relu") - args.dropout = getattr(args, "dropout", 0.1) - args.attention_dropout = getattr(args, "attention_dropout", 0.0) - args.activation_dropout = getattr(args, "activation_dropout", 0.0) - args.relu_dropout = getattr(args, "relu_dropout", 0.0) - args.decoder_embed_dim = getattr(args, "decoder_embed_dim", 512) - args.decoder_output_dim = getattr( - args, "decoder_output_dim", args.decoder_embed_dim - ) - args.decoder_input_dim = getattr(args, "decoder_input_dim", args.decoder_embed_dim) - args.decoder_ffn_embed_dim = getattr(args, "decoder_ffn_embed_dim", 2048) - args.decoder_layers = getattr(args, "decoder_layers", 6) - args.decoder_attention_heads = getattr(args, "decoder_attention_heads", 8) - # Model training is not stable without this - args.decoder_normalize_before = True - args.no_decoder_final_norm = getattr(args, "no_decoder_final_norm", False) - args.adaptive_softmax_cutoff = getattr(args, "adaptive_softmax_cutoff", None) - args.adaptive_softmax_dropout = getattr(args, "adaptive_softmax_dropout", 0) - args.adaptive_softmax_factor = getattr(args, "adaptive_softmax_factor", 4) - args.no_token_positional_embeddings = getattr( - args, "no_token_positional_embeddings", False - ) - args.share_decoder_input_output_embed = getattr( - args, "share_decoder_input_output_embed", False - ) - args.character_embeddings = getattr(args, "character_embeddings", False) - args.character_filters = getattr( - args, - "character_filters", - "[(1, 64), (2, 128), (3, 192), (4, 256), (5, 256), (6, 256), (7, 256)]", - ) - args.character_embedding_dim = getattr(args, "character_embedding_dim", 4) - args.char_embedder_highway_layers = getattr(args, "char_embedder_highway_layers", 2) - args.adaptive_input = getattr(args, "adaptive_input", False) - args.adaptive_input_factor = getattr(args, "adaptive_input_factor", 4) - args.adaptive_input_cutoff = getattr(args, "adaptive_input_cutoff", None) - args.tie_adaptive_weights = getattr(args, "tie_adaptive_weights", False) - args.tie_adaptive_proj = getattr(args, "tie_adaptive_proj", False) - args.decoder_learned_pos = getattr(args, "decoder_learned_pos", False) - args.decoder_layerdrop = getattr(args, "decoder_layerdrop", 0.0) - args.decoder_layers_to_keep = getattr(args, "decoder_layers_to_keep", None) - args.layernorm_embedding = getattr(args, "layernorm_embedding", False) - args.no_scale_embedding = getattr(args, "no_scale_embedding", False) - args.quant_noise_pq = getattr(args, "quant_noise_pq", 0.0) - args.quant_noise_pq_block_size = getattr(args, "quant_noise_pq_block_size", 8) - args.quant_noise_scalar = getattr(args, "quant_noise_scalar", 0.0) - args.add_bos_token = getattr(args, "add_bos_token", False) - - -@register_model_architecture("model_parallel_transformer_lm", "transformer_lm_megatron") -def transformer_lm_megatron(args): - args.decoder_embed_dim = getattr(args, "decoder_embed_dim", 3072) - args.decoder_ffn_embed_dim = getattr(args, "decoder_ffn_embed_dim", 3072 * 4) - args.decoder_layers = getattr(args, "decoder_layers", 72) - args.decoder_attention_heads = getattr(args, "decoder_attention_heads", 32) - args.dropout = getattr(args, "dropout", 0.1) - args.attention_dropout = getattr(args, "attention_dropout", 0.1) - args.activation_fn = getattr(args, "activation_fn", "gelu") - base_lm_architecture(args) - - -@register_model_architecture( - "model_parallel_transformer_lm", "transformer_lm_megatron_11b" -) -def transformer_lm_megatron_11b(args): - args.decoder_embed_dim = getattr(args, "decoder_embed_dim", 3072) - args.decoder_ffn_embed_dim = getattr(args, "decoder_ffn_embed_dim", 3072 * 6) - args.decoder_layers = getattr(args, "decoder_layers", 72) - args.decoder_attention_heads = getattr(args, "decoder_attention_heads", 32) - args.dropout = getattr(args, "dropout", 0.1) - args.attention_dropout = getattr(args, "attention_dropout", 0.1) - args.activation_fn = getattr(args, "activation_fn", "gelu") - base_lm_architecture(args) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/model_parallel/modules/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/model_parallel/modules/__init__.py deleted file mode 100644 index 11603217a188f420ea849ae0fde19979736ba208..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/model_parallel/modules/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. -"""isort:skip_file""" - -from .multihead_attention import ModelParallelMultiheadAttention -from .transformer_layer import ( - ModelParallelTransformerEncoderLayer, - ModelParallelTransformerDecoderLayer, -) - -__all__ = [ - "ModelParallelMultiheadAttention", - "ModelParallelTransformerEncoderLayer", - "ModelParallelTransformerDecoderLayer", -] diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/model_parallel/modules/multihead_attention.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/model_parallel/modules/multihead_attention.py deleted file mode 100644 index 721ff0b300c4f0f09679b1abf3b1b1822a74aa0a..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/model_parallel/modules/multihead_attention.py +++ /dev/null @@ -1,348 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from typing import Dict, Optional, Tuple - -import torch -import torch.nn.functional as F -from fairseq import utils -from fairseq.incremental_decoding_utils import with_incremental_state -from fairseq.modules.fairseq_dropout import FairseqDropout -from torch import Tensor, nn - -try: - from fairseq.model_parallel.megatron.mpu import ( - ColumnParallelLinear, - RowParallelLinear, - get_cuda_rng_tracker, - get_model_parallel_world_size, - ) - - has_megatron_submodule = True -except (ImportError, ModuleNotFoundError): - has_megatron_submodule = False - - -@with_incremental_state -class ModelParallelMultiheadAttention(nn.Module): - """Model parallel Multi-headed attention. - This performs the Multi-headed attention over multiple gpus. - - See "Megatron-LM: https://arxiv.org/pdf/1909.08053.pdf" for more details. - """ - - def __init__( - self, - embed_dim, - num_heads, - kdim=None, - vdim=None, - dropout=0.0, - bias=True, - self_attention=False, - encoder_decoder_attention=False, - ): - super().__init__() - if not has_megatron_submodule: - raise ImportError( - "\n\nPlease install the megatron submodule:" - "\n\n git submodule update --init " - "fairseq/model_parallel/megatron" - ) - self.embed_dim = embed_dim - self.kdim = kdim if kdim is not None else embed_dim - self.vdim = vdim if vdim is not None else embed_dim - self.qkv_same_dim = self.kdim == embed_dim and self.vdim == embed_dim - - self.model_parallel_size = get_model_parallel_world_size() - - self.num_heads_partition = num_heads // self.model_parallel_size - assert ( - self.num_heads_partition * self.model_parallel_size == num_heads - ), "Number of heads must be divisible by model parallel size" - - self.dropout_module = FairseqDropout( - dropout, module_name=self.__class__.__name__ - ) - self.head_dim = embed_dim // num_heads - assert ( - self.head_dim * num_heads == self.embed_dim - ), "embed_dim must be divisible by num_heads" - self.scaling = self.head_dim**-0.5 - - self.self_attention = self_attention - self.encoder_decoder_attention = encoder_decoder_attention - - assert ( - not self.self_attention or self.qkv_same_dim - ), "Self-attention requires query, key and value to be of the same size" - - self.k_proj = ColumnParallelLinear( - self.kdim, embed_dim, bias=bias, gather_output=False - ) - self.v_proj = ColumnParallelLinear( - self.vdim, embed_dim, bias=bias, gather_output=False - ) - self.q_proj = ColumnParallelLinear( - embed_dim, embed_dim, bias=bias, gather_output=False - ) - self.out_proj = RowParallelLinear( - embed_dim, embed_dim, bias=bias, input_is_parallel=True - ) - - def forward( - self, - query, - key: Optional[Tensor], - value: Optional[Tensor], - key_padding_mask: Optional[Tensor] = None, - incremental_state: Optional[Dict[str, Dict[str, Optional[Tensor]]]] = None, - static_kv: bool = False, - attn_mask: Optional[Tensor] = None, - **unused_kwargs, - ) -> Tuple[Tensor, Optional[Tensor]]: - """Input shape: Time x Batch x Channel - - Args: - key_padding_mask (ByteTensor, optional): mask to exclude - keys that are pads, of shape `(batch, src_len)`, where - padding elements are indicated by 1s. - attn_mask (ByteTensor, optional): typically used to - implement causal attention, where the mask prevents the - attention from looking forward in time (default: None). - """ - tgt_len, bsz, embed_dim = query.size() - assert embed_dim == self.embed_dim - assert list(query.size()) == [tgt_len, bsz, embed_dim] - - is_tpu = query.device.type == "xla" - - if incremental_state is not None: - saved_state = self._get_input_buffer(incremental_state) - if saved_state is not None and "prev_key" in saved_state: - # previous time steps are cached - no need to recompute - # key and value if they are static - if static_kv: - assert self.encoder_decoder_attention and not self.self_attention - key = value = None - else: - saved_state = None - - if self.self_attention: - q = self.q_proj(query) - k = self.k_proj(query) - v = self.v_proj(query) - elif self.encoder_decoder_attention: - # encoder-decoder attention - q = self.q_proj(query) - if key is None: - assert value is None - k = v = None - else: - k = self.k_proj(key) - v = self.v_proj(key) - - else: - assert key is not None and value is not None - q = self.q_proj(query) - k = self.k_proj(key) - v = self.v_proj(value) - q *= self.scaling - - q = ( - q.contiguous() - .view(tgt_len, bsz * self.num_heads_partition, self.head_dim) - .transpose(0, 1) - ) - if k is not None: - k = ( - k.contiguous() - .view(-1, bsz * self.num_heads_partition, self.head_dim) - .transpose(0, 1) - ) - if v is not None: - v = ( - v.contiguous() - .view(-1, bsz * self.num_heads_partition, self.head_dim) - .transpose(0, 1) - ) - - if saved_state is not None: - # saved states are stored with shape (bsz, num_heads_partition, seq_len, head_dim) - if "prev_key" in saved_state: - _prev_key = saved_state["prev_key"] - assert _prev_key is not None - prev_key = _prev_key.view( - bsz * self.num_heads_partition, -1, self.head_dim - ) - if static_kv: - k = prev_key - else: - assert k is not None - k = torch.cat([prev_key, k], dim=1) - if "prev_value" in saved_state: - _prev_value = saved_state["prev_value"] - assert _prev_value is not None - prev_value = _prev_value.view( - bsz * self.num_heads_partition, -1, self.head_dim - ) - if static_kv: - v = prev_value - else: - assert v is not None - v = torch.cat([prev_value, v], dim=1) - prev_key_padding_mask: Optional[Tensor] = None - if "prev_key_padding_mask" in saved_state: - prev_key_padding_mask = saved_state["prev_key_padding_mask"] - assert k is not None and v is not None - key_padding_mask = ( - ModelParallelMultiheadAttention._append_prev_key_padding_mask( - key_padding_mask=key_padding_mask, - prev_key_padding_mask=prev_key_padding_mask, - batch_size=bsz, - src_len=k.size(1), - static_kv=static_kv, - ) - ) - - saved_state["prev_key"] = k.view( - bsz, self.num_heads_partition, -1, self.head_dim - ) - saved_state["prev_value"] = v.view( - bsz, self.num_heads_partition, -1, self.head_dim - ) - saved_state["prev_key_padding_mask"] = key_padding_mask - # In this branch incremental_state is never None - assert incremental_state is not None - incremental_state = self._set_input_buffer(incremental_state, saved_state) - assert k is not None - src_len = k.size(1) - - # This is part of a workaround to get around fork/join parallelism - # not supporting Optional types. - if key_padding_mask is not None and key_padding_mask.dim() == 0: - key_padding_mask = None - - if key_padding_mask is not None: - assert key_padding_mask.size(0) == bsz - assert key_padding_mask.size(1) == src_len - - attn_weights = torch.bmm(q, k.transpose(1, 2)) - - assert list(attn_weights.size()) == [ - bsz * self.num_heads_partition, - tgt_len, - src_len, - ] - - if attn_mask is not None: - attn_mask = attn_mask.unsqueeze(0) - attn_weights += attn_mask - - if key_padding_mask is not None: - # don't attend to padding symbols - attn_weights = attn_weights.view( - bsz, self.num_heads_partition, tgt_len, src_len - ) - if not is_tpu: - attn_weights = attn_weights.masked_fill( - key_padding_mask.unsqueeze(1).unsqueeze(2).to(torch.bool), - float("-inf"), - ) - else: - attn_weights = attn_weights.transpose(0, 2) - attn_weights = attn_weights.masked_fill(key_padding_mask, float("-inf")) - attn_weights = attn_weights.transpose(0, 2) - attn_weights = attn_weights.view( - bsz * self.num_heads_partition, tgt_len, src_len - ) - - attn_weights_float = utils.softmax(attn_weights, dim=-1) - attn_weights = attn_weights_float.type_as(attn_weights) - - with get_cuda_rng_tracker().fork(): - attn_probs = self.dropout_module(attn_weights) - - assert v is not None - attn = torch.bmm(attn_probs, v) - assert list(attn.size()) == [ - bsz * self.num_heads_partition, - tgt_len, - self.head_dim, - ] - embed_dim_partition = embed_dim // self.model_parallel_size - attn = attn.transpose(0, 1).contiguous().view(tgt_len, bsz, embed_dim_partition) - attn = self.out_proj(attn) - # return attn_weights None to keep the return type same as single gpu multihead attention - # This will be deprecated. - attn_weights: Optional[Tensor] = None - - return attn, attn_weights - - @staticmethod - def _append_prev_key_padding_mask( - key_padding_mask: Optional[Tensor], - prev_key_padding_mask: Optional[Tensor], - batch_size: int, - src_len: int, - static_kv: bool, - ) -> Optional[Tensor]: - # saved key padding masks have shape (bsz, seq_len) - if prev_key_padding_mask is not None and static_kv: - new_key_padding_mask = prev_key_padding_mask - elif prev_key_padding_mask is not None and key_padding_mask is not None: - new_key_padding_mask = torch.cat( - [prev_key_padding_mask.float(), key_padding_mask.float()], dim=1 - ) - # During incremental decoding, as the padding token enters and - # leaves the frame, there will be a time when prev or current - # is None - elif prev_key_padding_mask is not None: - - filler = torch.zeros(batch_size, src_len - prev_key_padding_mask.size(1)) - if prev_key_padding_mask.is_cuda: - filler = filler.cuda() - new_key_padding_mask = torch.cat( - [prev_key_padding_mask.float(), filler.float()], dim=1 - ) - elif key_padding_mask is not None: - filler = torch.zeros(batch_size, src_len - key_padding_mask.size(1)) - if key_padding_mask.is_cuda: - filler = filler.cuda() - new_key_padding_mask = torch.cat( - [filler.float(), key_padding_mask.float()], dim=1 - ) - else: - new_key_padding_mask = prev_key_padding_mask - return new_key_padding_mask - - def reorder_incremental_state( - self, incremental_state: Dict[str, Dict[str, Optional[Tensor]]], new_order - ): - """Reorder buffered internal state (for incremental generation).""" - input_buffer = self._get_input_buffer(incremental_state) - if input_buffer is not None: - for k in input_buffer.keys(): - if input_buffer[k] is not None: - input_buffer[k] = input_buffer[k].index_select(0, new_order) - incremental_state = self._set_input_buffer(incremental_state, input_buffer) - return incremental_state - - def _get_input_buffer( - self, incremental_state: Optional[Dict[str, Dict[str, Optional[Tensor]]]] - ) -> Dict[str, Optional[Tensor]]: - result = self.get_incremental_state(incremental_state, "attn_state") - if result is not None: - return result - else: - empty_result: Dict[str, Optional[Tensor]] = {} - return empty_result - - def _set_input_buffer( - self, - incremental_state: Dict[str, Dict[str, Optional[Tensor]]], - buffer: Dict[str, Optional[Tensor]], - ): - return self.set_incremental_state(incremental_state, "attn_state", buffer) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/model_parallel/modules/transformer_layer.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/model_parallel/modules/transformer_layer.py deleted file mode 100644 index 7d438fba6caae5a81f11c3ae99c2b3298201f76e..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/model_parallel/modules/transformer_layer.py +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from fairseq.model_parallel.modules import ModelParallelMultiheadAttention -from fairseq.modules import TransformerDecoderLayer, TransformerEncoderLayer - -try: - from fairseq.model_parallel.megatron.mpu import ( - ColumnParallelLinear, - RowParallelLinear, - ) - - has_megatron_submodule = True -except (ImportError, ModuleNotFoundError): - has_megatron_submodule = False - - -class ModelParallelTransformerEncoderLayer(TransformerEncoderLayer): - """Encoder layer block over multiple gpus. - - See "Megatron-LM: https://arxiv.org/pdf/1909.08053.pdf" for more details. - """ - - def build_fc1(self, input_dim, output_dim, q_noise, qn_block_size): - if q_noise > 0: - raise NotImplementedError - return ColumnParallelLinear(input_dim, output_dim, gather_output=False) - - def build_fc2(self, input_dim, output_dim, q_noise, qn_block_size): - if q_noise > 0: - raise NotImplementedError - return RowParallelLinear(input_dim, output_dim, input_is_parallel=True) - - def build_self_attention(self, embed_dim, args, **unused_kwargs): - return ModelParallelMultiheadAttention( - embed_dim, - args.encoder_attention_heads, - dropout=args.attention_dropout, - self_attention=True, - ) - - -class ModelParallelTransformerDecoderLayer(TransformerDecoderLayer): - """Decoder layer block. - - See "Megatron-LM: https://arxiv.org/pdf/1909.08053.pdf" for more details. - """ - - def build_fc1(self, input_dim, output_dim, q_noise, qn_block_size): - if q_noise > 0: - raise NotImplementedError - return ColumnParallelLinear(input_dim, output_dim, gather_output=False) - - def build_fc2(self, input_dim, output_dim, q_noise, qn_block_size): - if q_noise > 0: - raise NotImplementedError - return RowParallelLinear(input_dim, output_dim, input_is_parallel=True) - - def build_self_attention(self, embed_dim, args, **unused_kwargs): - return ModelParallelMultiheadAttention( - embed_dim=embed_dim, - num_heads=args.decoder_attention_heads, - dropout=args.attention_dropout, - self_attention=not getattr(args, "cross_self_attention", False), - ) - - def build_encoder_attention(self, embed_dim, args, **unused_kwargs): - return ModelParallelMultiheadAttention( - embed_dim=embed_dim, - num_heads=args.decoder_attention_heads, - kdim=getattr(args, "encoder_embed_dim", None), - vdim=getattr(args, "encoder_embed_dim", None), - dropout=args.attention_dropout, - encoder_decoder_attention=True, - ) diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/models/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/models/__init__.py deleted file mode 100644 index 616e3051e9a733fa1a821f88b375052a7589d340..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/models/__init__.py +++ /dev/null @@ -1,235 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. -"""isort:skip_file""" - -import argparse -import importlib -import os - -from contextlib import ExitStack - -from fairseq.dataclass import FairseqDataclass -from fairseq.dataclass.utils import merge_with_parent -from hydra.core.config_store import ConfigStore -from omegaconf import open_dict, OmegaConf - -from .composite_encoder import CompositeEncoder -from .distributed_fairseq_model import DistributedFairseqModel -from .fairseq_decoder import FairseqDecoder -from .fairseq_encoder import FairseqEncoder -from .fairseq_incremental_decoder import FairseqIncrementalDecoder -from .fairseq_model import ( - BaseFairseqModel, - FairseqEncoderDecoderModel, - FairseqEncoderModel, - FairseqLanguageModel, - FairseqModel, - FairseqMultiModel, -) - - -MODEL_REGISTRY = {} -MODEL_DATACLASS_REGISTRY = {} -ARCH_MODEL_REGISTRY = {} -ARCH_MODEL_NAME_REGISTRY = {} -ARCH_MODEL_INV_REGISTRY = {} -ARCH_CONFIG_REGISTRY = {} - - -__all__ = [ - "BaseFairseqModel", - "CompositeEncoder", - "DistributedFairseqModel", - "FairseqDecoder", - "FairseqEncoder", - "FairseqEncoderDecoderModel", - "FairseqEncoderModel", - "FairseqIncrementalDecoder", - "FairseqLanguageModel", - "FairseqModel", - "FairseqMultiModel", -] - - -def build_model(cfg: FairseqDataclass, task, from_checkpoint=False): - - model = None - model_type = getattr(cfg, "_name", None) or getattr(cfg, "arch", None) - - if not model_type and len(cfg) == 1: - # this is hit if config object is nested in directory that is named after model type - - model_type = next(iter(cfg)) - if model_type in MODEL_DATACLASS_REGISTRY: - cfg = cfg[model_type] - else: - raise Exception( - "Could not infer model type from directory. Please add _name field to indicate model type. " - "Available models: " - + str(MODEL_DATACLASS_REGISTRY.keys()) - + " Requested model type: " - + model_type - ) - - if model_type in ARCH_MODEL_REGISTRY: - # case 1: legacy models - model = ARCH_MODEL_REGISTRY[model_type] - elif model_type in MODEL_DATACLASS_REGISTRY: - # case 2: config-driven models - model = MODEL_REGISTRY[model_type] - - if model_type in MODEL_DATACLASS_REGISTRY: - # set defaults from dataclass. note that arch name and model name can be the same - dc = MODEL_DATACLASS_REGISTRY[model_type] - - if isinstance(cfg, argparse.Namespace): - cfg = dc.from_namespace(cfg) - else: - cfg = merge_with_parent(dc(), cfg, from_checkpoint) - else: - if model_type in ARCH_CONFIG_REGISTRY: - with open_dict(cfg) if OmegaConf.is_config(cfg) else ExitStack(): - # this calls the different "arch" functions (like base_architecture()) that you indicate - # if you specify --arch on the command line. this is only applicable to the old argparse based models - # hydra models should expose different architectures via different config files - # it will modify the cfg object and default parameters according to the arch - ARCH_CONFIG_REGISTRY[model_type](cfg) - - assert model is not None, ( - f"Could not infer model type from {cfg}. " - "Available models: {}".format(MODEL_DATACLASS_REGISTRY.keys()) - + f" Requested model type: {model_type}" - ) - - return model.build_model(cfg, task) - - -def register_model(name, dataclass=None): - """ - New model types can be added to fairseq with the :func:`register_model` - function decorator. - - For example:: - - @register_model('lstm') - class LSTM(FairseqEncoderDecoderModel): - (...) - - .. note:: All models must implement the :class:`BaseFairseqModel` interface. - Typically you will extend :class:`FairseqEncoderDecoderModel` for - sequence-to-sequence tasks or :class:`FairseqLanguageModel` for - language modeling tasks. - - Args: - name (str): the name of the model - """ - - def register_model_cls(cls): - if name in MODEL_REGISTRY: - raise ValueError("Cannot register duplicate model ({})".format(name)) - if not issubclass(cls, BaseFairseqModel): - raise ValueError( - "Model ({}: {}) must extend BaseFairseqModel".format(name, cls.__name__) - ) - MODEL_REGISTRY[name] = cls - if dataclass is not None and not issubclass(dataclass, FairseqDataclass): - raise ValueError( - "Dataclass {} must extend FairseqDataclass".format(dataclass) - ) - - cls.__dataclass = dataclass - if dataclass is not None: - MODEL_DATACLASS_REGISTRY[name] = dataclass - - cs = ConfigStore.instance() - node = dataclass() - node._name = name - cs.store(name=name, group="model", node=node, provider="fairseq") - - @register_model_architecture(name, name) - def noop(_): - pass - - return cls - - return register_model_cls - - -def register_model_architecture(model_name, arch_name): - """ - New model architectures can be added to fairseq with the - :func:`register_model_architecture` function decorator. After registration, - model architectures can be selected with the ``--arch`` command-line - argument. - - For example:: - - @register_model_architecture('lstm', 'lstm_luong_wmt_en_de') - def lstm_luong_wmt_en_de(cfg): - args.encoder_embed_dim = getattr(cfg.model, 'encoder_embed_dim', 1000) - (...) - - The decorated function should take a single argument *cfg*, which is a - :class:`omegaconf.DictConfig`. The decorated function should modify these - arguments in-place to match the desired architecture. - - Args: - model_name (str): the name of the Model (Model must already be - registered) - arch_name (str): the name of the model architecture (``--arch``) - """ - - def register_model_arch_fn(fn): - if model_name not in MODEL_REGISTRY: - raise ValueError( - "Cannot register model architecture for unknown model type ({})".format( - model_name - ) - ) - if arch_name in ARCH_MODEL_REGISTRY: - raise ValueError( - "Cannot register duplicate model architecture ({})".format(arch_name) - ) - if not callable(fn): - raise ValueError( - "Model architecture must be callable ({})".format(arch_name) - ) - ARCH_MODEL_REGISTRY[arch_name] = MODEL_REGISTRY[model_name] - ARCH_MODEL_NAME_REGISTRY[arch_name] = model_name - ARCH_MODEL_INV_REGISTRY.setdefault(model_name, []).append(arch_name) - ARCH_CONFIG_REGISTRY[arch_name] = fn - return fn - - return register_model_arch_fn - - -def import_models(models_dir, namespace): - for file in os.listdir(models_dir): - path = os.path.join(models_dir, file) - if ( - not file.startswith("_") - and not file.startswith(".") - and (file.endswith(".py") or os.path.isdir(path)) - ): - model_name = file[: file.find(".py")] if file.endswith(".py") else file - importlib.import_module(namespace + "." + model_name) - - # extra `model_parser` for sphinx - if model_name in MODEL_REGISTRY: - parser = argparse.ArgumentParser(add_help=False) - group_archs = parser.add_argument_group("Named architectures") - group_archs.add_argument( - "--arch", choices=ARCH_MODEL_INV_REGISTRY[model_name] - ) - group_args = parser.add_argument_group( - "Additional command-line arguments" - ) - MODEL_REGISTRY[model_name].add_args(group_args) - globals()[model_name + "_parser"] = parser - - -# automatically import any Python files in the models/ directory -models_dir = os.path.dirname(__file__) -import_models(models_dir, "fairseq.models") diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/models/bart/__init__.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/models/bart/__init__.py deleted file mode 100644 index a701923f7e5a2a8aa9b75e5580ddea22907f53ee..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/models/bart/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from .hub_interface import * # noqa -from .model import * # noqa diff --git a/audio/safeear/safeear_repo/fairseq_ours/fairseq/models/bart/hub_interface.py b/audio/safeear/safeear_repo/fairseq_ours/fairseq/models/bart/hub_interface.py deleted file mode 100644 index 9e92255c7666d51f2652a58d2b45b9491851ee00..0000000000000000000000000000000000000000 --- a/audio/safeear/safeear_repo/fairseq_ours/fairseq/models/bart/hub_interface.py +++ /dev/null @@ -1,210 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import copy -import logging -from typing import Dict, List - -import numpy as np -import torch -import torch.nn as nn -import torch.nn.functional as F -from fairseq import utils -from fairseq.data import encoders -from fairseq.hub_utils import GeneratorHubInterface -from omegaconf import open_dict - -logger = logging.getLogger(__name__) - - -class BARTHubInterface(GeneratorHubInterface): - """A simple PyTorch Hub interface to BART. - - Usage: https://github.com/pytorch/fairseq/tree/main/examples/bart - """ - - def __init__(self, cfg, task, model): - super().__init__(cfg, task, [model]) - self.model = self.models[0] - - def encode( - self, sentence: str, *addl_sentences, no_separator=True - ) -> torch.LongTensor: - """ - BPE-encode a sentence (or multiple sentences). - - Every sequence begins with a beginning-of-sentence (`<s>`) symbol. - Every sentence ends with an end-of-sentence (`</s>`). - - Example (single sentence): `<s> a b c </s>` - Example (sentence pair): `<s> d e f </s> 1 2 3 </s>` - - The BPE encoding follows GPT-2. One subtle detail is that the GPT-2 BPE - requires leading spaces. For example:: - - >>> bart.encode('Hello world').tolist() - [0, 31414, 232, 2] - >>> bart.encode(' world').tolist() - [0, 232, 2] - >>> bart.encode('world').tolist() - [0, 8331, 2] - """ - tokens = self.bpe.encode(sentence) - if len(tokens.split(" ")) > min(self.max_positions) - 2: - tokens = " ".join(tokens.split(" ")[: min(self.max_positions) - 2]) - bpe_sentence = "<s> " + tokens + " </s>" - for s in addl_sentences: - bpe_sentence += " </s>" if not no_separator else "" - bpe_sentence += " " + self.bpe.encode(s) + " </s>" - tokens = self.task.source_dictionary.encode_line(bpe_sentence, append_eos=False) - return tokens.long() - - def decode(self, tokens: torch.LongTensor): - assert tokens.dim() == 1 - tokens = tokens.cpu().numpy() - if tokens[0] == self.task.source_dictionary.bos(): - tokens = tokens[1:] # remove <s> - eos_mask = tokens == self.task.source_dictionary.eos() - doc_mask = eos_mask[1:] & eos_mask[:-1] - sentences = np.split(tokens, doc_mask.nonzero()[0] + 1) - sentences = [ - self.bpe.decode(self.task.source_dictionary.string(s)) for s in sentences - ] - if len(sentences) == 1: - return sentences[0] - return sentences - - def _build_sample(self, src_tokens: List[torch.LongTensor]): - # assert torch.is_tensor(src_tokens) - dataset = self.task.build_dataset_for_inference( - src_tokens, - [x.numel() for x in src_tokens], - ) - sample = dataset.collater(dataset) - sample = utils.apply_to_sample(lambda tensor: tensor.to(self.device), sample) - return sample - - def generate( - self, - tokenized_sentences: List[torch.LongTensor], - *args, - inference_step_args=None, - skip_invalid_size_inputs=False, - **kwargs - ) -> List[List[Dict[str, torch.Tensor]]]: - inference_step_args = inference_step_args or {} - if "prefix_tokens" in inference_step_args: - raise NotImplementedError("prefix generation not implemented for BART") - res = [] - for batch in self._build_batches(tokenized_sentences, skip_invalid_size_inputs): - src_tokens = batch["net_input"]["src_tokens"] - inference_step_args["prefix_tokens"] = src_tokens.new_full( - (src_tokens.size(0), 1), fill_value=self.task.source_dictionary.bos() - ).to(device=self.device) - results = super().generate( - src_tokens, - *args, - inference_step_args=inference_step_args, - skip_invalid_size_inputs=skip_invalid_size_inputs, - **kwargs - ) - for id, hypos in zip(batch["id"].tolist(), results): - res.append((id, hypos)) - res = [hypos for _, hypos in sorted(res, key=lambda x: x[0])] - return res - - def extract_features( - self, tokens: torch.LongTensor, return_all_hiddens: bool = False - ) -> torch.Tensor: - if tokens.dim() == 1: - tokens = tokens.unsqueeze(0) - if tokens.size(-1) > min(self.model.max_positions()): - raise ValueError( - "tokens exceeds maximum length: {} > {}".format( - tokens.size(-1), self.model.max_positions() - ) - ) - tokens.to(device=self.device), - prev_output_tokens = tokens.clone() - - prev_output_tokens[:, 0] = tokens.gather( - 1, - (tokens.ne(self.task.source_dictionary.pad()).sum(dim=1) - 1).unsqueeze(-1), - ).squeeze() - - prev_output_tokens[:, 1:] = tokens[:, :-1] - features, extra = self.model( - src_tokens=tokens, - src_lengths=None, - prev_output_tokens=prev_output_tokens, - features_only=True, - return_all_hiddens=return_all_hiddens, - ) - if return_all_hiddens: - # convert from T x B x C -> B x T x C - inner_states = extra["inner_states"] - return [inner_state.transpose(0, 1) for inner_state in inner_states] - else: - return features # just the last layer's features - - def register_classification_head( - self, name: str, num_classes: int = None, embedding_size: int = None, **kwargs - ): - self.model.register_classification_head( - name, num_classes=num_classes, embedding_size=embedding_size, **kwargs - ) - - def predict(self, head: str, tokens: torch.LongTensor, return_logits: bool = False): - if tokens.dim() == 1: - tokens = tokens.unsqueeze(0) - features = self.extract_features(tokens.to(device=self.device)) - sentence_representation = features[ - tokens.eq(self.task.source_dictionary.eos()), : - ].view(features.size(0), -1, features.size(-1))[:, -1, :] - - logits = self.model.classification_heads[head](sentence_representation) - if return_logits: - return logits - return F.log_softmax(logits, dim=-1) - - def fill_mask( - self, - masked_inputs: List[str], - topk: int = 5, - match_source_len: bool = True, - **generate_kwargs - ): - masked_token = "<mask>" - batch_tokens = [] - for masked_input in masked_inputs: - assert ( - masked_token in masked_input - ), "please add one {} token for the input".format(masked_token) - - text_spans = masked_input.split(masked_token) - text_spans_bpe = ( - (" {0} ".format(masked_token)) - .join([self.bpe.encode(text_span.rstrip()) for text_span in text_spans]) - .strip() - ) - tokens = self.task.source_dictionary.encode_line( - "<s> " + text_spans_bpe + " </s>", - append_eos=False, - add_if_not_exist=False, - ).long() - batch_tokens.append(tokens) - - # ensure beam size is at least as big as topk - generate_kwargs["beam"] = max( - topk, - generate_kwargs.get("beam", -1), - ) - generate_kwargs["match_source_len"] = match_source_len - batch_hypos = self.generate(batch_tokens, **generate_kwargs) - - return [ - [(self.decode(hypo["tokens"]), hypo["score"]) for hypo in hypos[:topk]] - for hypos in batch_hypos - ]