chore: remove non-weight files (batch 2)

audio/safeear/safeear_repo/fairseq_ours/examples/simultaneous_translation/utils/monotonic_attention.py

@@ -1,180 +0,0 @@
-from typing import Optional
-
-import torch
-from examples.simultaneous_translation.utils.functions import (
-    exclusive_cumprod,
-    moving_sum,
-    prob_check,
-)
-from torch import Tensor
-
-
-def expected_alignment_from_p_choose(
-    p_choose: Tensor,
-    padding_mask: Optional[Tensor] = None,
-    eps: float = 1e-6
-):
-    """
-    Calculating expected alignment for from stepwise probability
-
-    Reference:
-    Online and Linear-Time Attention by Enforcing Monotonic Alignments
-    https://arxiv.org/pdf/1704.00784.pdf
-
-    q_ij = (1 − p_{ij−1})q_{ij−1} + a+{i−1j}
-    a_ij = p_ij q_ij
-
-    Parallel solution:
-    ai = p_i * cumprod(1 − pi) * cumsum(a_i / cumprod(1 − pi))
-
-    ============================================================
-    Expected input size
-    p_choose: bsz, tgt_len, src_len
-    """
-    prob_check(p_choose)
-
-    # p_choose: bsz, tgt_len, src_len
-    bsz, tgt_len, src_len = p_choose.size()
-    dtype = p_choose.dtype
-
-    p_choose = p_choose.float()
-
-    if padding_mask is not None:
-        p_choose = p_choose.masked_fill(padding_mask.unsqueeze(1), 0.0)
-
-    if p_choose.is_cuda:
-        p_choose = p_choose.contiguous()
-        from alignment_train_cuda_binding import alignment_train_cuda as alignment_train
-    else:
-        from alignment_train_cpu_binding import alignment_train_cpu as alignment_train
-
-    alpha = p_choose.new_zeros([bsz, tgt_len, src_len])
-    alignment_train(p_choose, alpha, eps)
-
-    # Mix precision to prevent overflow for fp16
-    alpha = alpha.type(dtype)
-
-    prob_check(alpha)
-
-    return alpha
-
-
-def expected_soft_attention(
-    alpha: Tensor,
-    soft_energy: Tensor,
-    padding_mask: Optional[Tensor] = None,
-    chunk_size: Optional[int] = None,
-    eps: float = 1e-10
-):
-    """
-    Function to compute expected soft attention for
-    monotonic infinite lookback attention from
-    expected alignment and soft energy.
-
-    Reference:
-    Monotonic Chunkwise Attention
-    https://arxiv.org/abs/1712.05382
-
-    Monotonic Infinite Lookback Attention for Simultaneous Machine Translation
-    https://arxiv.org/abs/1906.05218
-
-    alpha: bsz, tgt_len, src_len
-    soft_energy: bsz, tgt_len, src_len
-    padding_mask: bsz, src_len
-    left_padding: bool
-    """
-    if padding_mask is not None:
-        alpha = alpha.masked_fill(padding_mask.unsqueeze(1), 0.0)
-        soft_energy = soft_energy.masked_fill(
-            padding_mask.unsqueeze(1), -float("inf")
-        )
-
-    prob_check(alpha)
-
-    dtype = alpha.dtype
-
-    alpha = alpha.float()
-    soft_energy = soft_energy.float()
-
-    soft_energy = soft_energy - soft_energy.max(dim=2, keepdim=True)[0]
-    exp_soft_energy = torch.exp(soft_energy) + eps
-
-    if chunk_size is not None:
-        # Chunkwise
-        beta = (
-            exp_soft_energy
-            * moving_sum(
-                alpha / (eps + moving_sum(exp_soft_energy, chunk_size, 1)),
-                1, chunk_size
-            )
-        )
-    else:
-        # Infinite lookback
-        # Notice that infinite lookback is a special case of chunkwise
-        # where chunksize = inf
-        inner_items = alpha / (eps + torch.cumsum(exp_soft_energy, dim=2))
-
-        beta = (
-            exp_soft_energy
-            * torch.cumsum(inner_items.flip(dims=[2]), dim=2)
-            .flip(dims=[2])
-        )
-
-    if padding_mask is not None:
-        beta = beta.masked_fill(
-            padding_mask.unsqueeze(1).to(torch.bool), 0.0)
-
-    # Mix precision to prevent overflow for fp16
-    beta = beta.type(dtype)
-
-    beta = beta.clamp(0, 1)
-
-    prob_check(beta)
-
-    return beta
-
-
-def mass_preservation(
-    alpha: Tensor,
-    padding_mask: Optional[Tensor] = None,
-    left_padding: bool = False
-):
-    """
-    Function to compute the mass perservation for alpha.
-    This means that the residual weights of alpha will be assigned
-    to the last token.
-
-    Reference:
-    Monotonic Infinite Lookback Attention for Simultaneous Machine Translation
-    https://arxiv.org/abs/1906.05218
-
-    alpha: bsz, tgt_len, src_len
-    padding_mask: bsz, src_len
-    left_padding: bool
-    """
-
-    prob_check(alpha)
-
-    if padding_mask is not None:
-        if not left_padding:
-            assert not padding_mask[:, 0].any(), (
-                "Find padding on the beginning of the sequence."
-            )
-        alpha = alpha.masked_fill(padding_mask.unsqueeze(1), 0.0)
-
-    if left_padding or padding_mask is None:
-        residuals = 1 - alpha[:, :, :-1].sum(dim=-1).clamp(0, 1)
-        alpha[:, :, -1] = residuals
-    else:
-        # right padding
-        _, tgt_len, src_len = alpha.size()
-        residuals = 1 - alpha.sum(dim=-1, keepdim=True).clamp(0, 1)
-        src_lens = src_len - padding_mask.sum(dim=1, keepdim=True)
-        src_lens = src_lens.expand(-1, tgt_len).contiguous()
-        # add back the last value
-        residuals += alpha.gather(2, src_lens.unsqueeze(2) - 1)
-        alpha = alpha.scatter(2, src_lens.unsqueeze(2) - 1, residuals)
-
-        prob_check(alpha)
-
-    return alpha

audio/safeear/safeear_repo/fairseq_ours/examples/simultaneous_translation/utils/p_choose_strategy.py

@@ -1,127 +0,0 @@
-from typing import Dict, Optional
-
-import torch
-from torch import Tensor
-
-
-def waitk_p_choose(
-    tgt_len: int,
-    src_len: int,
-    bsz: int,
-    waitk_lagging: int,
-    key_padding_mask: Optional[Tensor] = None,
-    incremental_state: Optional[Dict[str, Dict[str, Optional[Tensor]]]] = None
-):
-
-    max_src_len = src_len
-    if incremental_state is not None:
-        # Retrieve target length from incremental states
-        # For inference the length of query is always 1
-        max_tgt_len = incremental_state["steps"]["tgt"]
-        assert max_tgt_len is not None
-        max_tgt_len = int(max_tgt_len)
-    else:
-        max_tgt_len = tgt_len
-
-    if max_src_len < waitk_lagging:
-        if incremental_state is not None:
-            max_tgt_len = 1
-        return torch.zeros(
-            bsz, max_tgt_len, max_src_len
-        )
-
-    # Assuming the p_choose looks like this for wait k=3
-    # src_len = 6, max_tgt_len = 5
-    #   [0, 0, 1, 0, 0, 0, 0]
-    #   [0, 0, 0, 1, 0, 0, 0]
-    #   [0, 0, 0, 0, 1, 0, 0]
-    #   [0, 0, 0, 0, 0, 1, 0]
-    #   [0, 0, 0, 0, 0, 0, 1]
-    # linearize the p_choose matrix:
-    # [0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0...]
-    # The indices of linearized matrix that equals 1 is
-    # 2 + 6 * 0
-    # 3 + 6 * 1
-    # ...
-    # n + src_len * n + k - 1 = n * (src_len + 1) + k - 1
-    # n from 0 to max_tgt_len - 1
-    #
-    # First, generate the indices (activate_indices_offset: bsz, max_tgt_len)
-    # Second, scatter a zeros tensor (bsz, max_tgt_len * src_len)
-    # with activate_indices_offset
-    # Third, resize the tensor to (bsz, max_tgt_len, src_len)
-
-    activate_indices_offset = (
-        (
-            torch.arange(max_tgt_len) * (max_src_len + 1)
-            + waitk_lagging - 1
-        )
-        .unsqueeze(0)
-        .expand(bsz, max_tgt_len)
-        .long()
-    )
-
-    if key_padding_mask is not None:
-        if key_padding_mask[:, 0].any():
-            # Left padding
-            activate_indices_offset += (
-                key_padding_mask.sum(dim=1, keepdim=True)
-            )
-
-    # Need to clamp the indices that are too large
-    activate_indices_offset = (
-        activate_indices_offset
-        .clamp(
-            0,
-            min(
-                [
-                    max_tgt_len,
-                    max_src_len - waitk_lagging + 1
-                ]
-            ) * max_src_len - 1
-        )
-    )
-
-    p_choose = torch.zeros(bsz, max_tgt_len * max_src_len)
-
-    p_choose = p_choose.scatter(
-        1,
-        activate_indices_offset,
-        1.0
-    ).view(bsz, max_tgt_len, max_src_len)
-
-    if key_padding_mask is not None:
-        p_choose = p_choose.to(key_padding_mask)
-        p_choose = p_choose.masked_fill(key_padding_mask.unsqueeze(1), 0)
-
-    if incremental_state is not None:
-        p_choose = p_choose[:, -1:]
-
-    return p_choose.float()
-
-
-def learnable_p_choose(
-    energy,
-    noise_mean: float = 0.0,
-    noise_var: float = 0.0,
-    training: bool = True
-):
-    """
-    Calculating step wise prob for reading and writing
-    1 to read, 0 to write
-    energy: bsz, tgt_len, src_len
-    """
-
-    noise = 0
-    if training:
-        # add noise here to encourage discretness
-        noise = (
-            torch.normal(noise_mean, noise_var, energy.size())
-            .type_as(energy)
-            .to(energy.device)
-        )
-
-    p_choose = torch.sigmoid(energy + noise)
-
-    # p_choose: bsz * self.num_heads, tgt_len, src_len
-    return p_choose

audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/README.md

@@ -1,83 +0,0 @@
-### 2021 Update: We are merging this example into the [S2T framework](../speech_to_text), which supports more generic speech-to-text tasks (e.g. speech translation) and more flexible data processing pipelines. Please stay tuned.
-
-# Speech Recognition
-`examples/speech_recognition` is implementing ASR task in Fairseq, along with needed features, datasets, models and loss functions to train and infer model described in [Transformers with convolutional context for ASR (Abdelrahman Mohamed et al., 2019)](https://arxiv.org/abs/1904.11660).
-
-
-## Additional dependencies
-On top of main fairseq dependencies there are couple more additional requirements.
-
-1) Please follow the instructions to install [torchaudio](https://github.com/pytorch/audio). This is required to compute audio fbank features.
-2) [Sclite](http://www1.icsi.berkeley.edu/Speech/docs/sctk-1.2/sclite.htm#sclite_name_0) is used to measure WER. Sclite can be downloaded and installed from source from sctk package [here](http://www.openslr.org/4/). Training and inference doesn't require Sclite dependency.
-3) [sentencepiece](https://github.com/google/sentencepiece) is required in order to create dataset with word-piece targets.
-
-## Preparing librispeech data
-```
-./examples/speech_recognition/datasets/prepare-librispeech.sh $DIR_TO_SAVE_RAW_DATA $DIR_FOR_PREPROCESSED_DATA
-```
-
-## Training librispeech data
-```
-python train.py $DIR_FOR_PREPROCESSED_DATA --save-dir $MODEL_PATH --max-epoch 80 --task speech_recognition --arch vggtransformer_2 --optimizer adadelta --lr 1.0 --adadelta-eps 1e-8 --adadelta-rho 0.95 --clip-norm 10.0  --max-tokens 5000 --log-format json --log-interval 1 --criterion cross_entropy_acc --user-dir examples/speech_recognition/
-```
-
-## Inference for librispeech
-`$SET` can be `test_clean` or `test_other`
-Any checkpoint in `$MODEL_PATH` can be selected. In this example we are working with `checkpoint_last.pt`
-```
-python examples/speech_recognition/infer.py $DIR_FOR_PREPROCESSED_DATA --task speech_recognition --max-tokens 25000 --nbest 1 --path $MODEL_PATH/checkpoint_last.pt --beam 20 --results-path $RES_DIR --batch-size 40 --gen-subset $SET --user-dir examples/speech_recognition/
-```
-
-## Inference for librispeech
-```
-sclite -r ${RES_DIR}/ref.word-checkpoint_last.pt-${SET}.txt -h ${RES_DIR}/hypo.word-checkpoint_last.pt-${SET}.txt -i rm -o all stdout > $RES_REPORT
-```
-`Sum/Avg` row from first table of the report has WER
-
-## Using flashlight (previously called [wav2letter](https://github.com/facebookresearch/wav2letter)) components
-[flashlight](https://github.com/facebookresearch/flashlight) now has integration with fairseq. Currently this includes:
-
-* AutoSegmentationCriterion (ASG)
-* flashlight-style Conv/GLU model
-* flashlight's beam search decoder
-
-To use these, follow the instructions on [this page](https://github.com/facebookresearch/flashlight/tree/master/bindings/python) to install python bindings.
-
-## Training librispeech data (flashlight style, Conv/GLU + ASG loss)
-Training command:
-```
-python train.py $DIR_FOR_PREPROCESSED_DATA --save-dir $MODEL_PATH --max-epoch 100 --task speech_recognition --arch w2l_conv_glu_enc --batch-size 4 --optimizer sgd --lr 0.3,0.8 --momentum 0.8 --clip-norm 0.2 --max-tokens 50000 --log-format json --log-interval 100 --num-workers 0 --sentence-avg --criterion asg_loss --asg-transitions-init 5 --max-replabel 2 --linseg-updates 8789 --user-dir examples/speech_recognition
-```
-
-Note that ASG loss currently doesn't do well with word-pieces. You should prepare a dataset with character targets by setting `nbpe=31` in `prepare-librispeech.sh`.
-
-## Inference for librispeech (flashlight decoder, n-gram LM)
-Inference command:
-```
-python examples/speech_recognition/infer.py $DIR_FOR_PREPROCESSED_DATA --task speech_recognition --seed 1 --nbest 1 --path $MODEL_PATH/checkpoint_last.pt --gen-subset $SET --results-path $RES_DIR --w2l-decoder kenlm --kenlm-model $KENLM_MODEL_PATH --lexicon $LEXICON_PATH --beam 200 --beam-threshold 15 --lm-weight 1.5 --word-score 1.5 --sil-weight -0.3 --criterion asg_loss --max-replabel 2 --user-dir examples/speech_recognition
-```
-
-`$KENLM_MODEL_PATH` should be a standard n-gram language model file. `$LEXICON_PATH` should be a flashlight-style lexicon (list of known words and their spellings). For ASG inference, a lexicon line should look like this (note the repetition labels):
-```
-doorbell  D O 1 R B E L 1 ▁
-```
-For CTC inference with word-pieces, repetition labels are not used and the lexicon should have most common spellings for each word (one can use sentencepiece's `NBestEncodeAsPieces` for this):
-```
-doorbell  ▁DOOR BE LL
-doorbell  ▁DOOR B E LL
-doorbell  ▁DO OR BE LL
-doorbell  ▁DOOR B EL L
-doorbell  ▁DOOR BE L L
-doorbell  ▁DO OR B E LL
-doorbell  ▁DOOR B E L L
-doorbell  ▁DO OR B EL L
-doorbell  ▁DO O R BE LL
-doorbell  ▁DO OR BE L L
-```
-Lowercase vs. uppercase matters: the *word* should match the case of the n-gram language model (i.e. `$KENLM_MODEL_PATH`), while the *spelling* should match the case of the token dictionary (i.e. `$DIR_FOR_PREPROCESSED_DATA/dict.txt`).
-
-## Inference for librispeech (flashlight decoder, viterbi only)
-Inference command:
-```
-python examples/speech_recognition/infer.py $DIR_FOR_PREPROCESSED_DATA --task speech_recognition --seed 1 --nbest 1 --path $MODEL_PATH/checkpoint_last.pt --gen-subset $SET --results-path $RES_DIR --w2l-decoder viterbi --criterion asg_loss --max-replabel 2 --user-dir examples/speech_recognition
-```

audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/__init__.py

@@ -1 +0,0 @@
-from . import criterions, models, tasks  # noqa

audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/criterions/ASG_loss.py

@@ -1,170 +0,0 @@
-#!/usr/bin/env python3
-
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-import torch
-from examples.speech_recognition.data.replabels import pack_replabels
-from fairseq import utils
-from fairseq.criterions import FairseqCriterion, register_criterion
-
-
-@register_criterion("asg_loss")
-class ASGCriterion(FairseqCriterion):
-    @staticmethod
-    def add_args(parser):
-        group = parser.add_argument_group("ASG Loss")
-        group.add_argument(
-            "--asg-transitions-init",
-            help="initial diagonal value of transition matrix",
-            type=float,
-            default=0.0,
-        )
-        group.add_argument(
-            "--max-replabel", help="maximum # of replabels", type=int, default=2
-        )
-        group.add_argument(
-            "--linseg-updates",
-            help="# of training updates to use LinSeg initialization",
-            type=int,
-            default=0,
-        )
-        group.add_argument(
-            "--hide-linseg-messages",
-            help="hide messages about LinSeg initialization",
-            action="store_true",
-        )
-
-    def __init__(
-        self,
-        task,
-        silence_token,
-        asg_transitions_init,
-        max_replabel,
-        linseg_updates,
-        hide_linseg_messages,
-    ):
-        from flashlight.lib.sequence.criterion import ASGLoss, CriterionScaleMode
-
-        super().__init__(task)
-        self.tgt_dict = task.target_dictionary
-        self.eos = self.tgt_dict.eos()
-        self.silence = (
-            self.tgt_dict.index(silence_token)
-            if silence_token in self.tgt_dict
-            else None
-        )
-        self.max_replabel = max_replabel
-
-        num_labels = len(self.tgt_dict)
-        self.asg = ASGLoss(num_labels, scale_mode=CriterionScaleMode.TARGET_SZ_SQRT)
-        self.asg.trans = torch.nn.Parameter(
-            asg_transitions_init * torch.eye(num_labels), requires_grad=True
-        )
-
-        self.linseg_progress = torch.nn.Parameter(
-            torch.tensor([0], dtype=torch.int), requires_grad=False
-        )
-        self.linseg_maximum = linseg_updates
-        self.linseg_message_state = "none" if hide_linseg_messages else "start"
-
-    @classmethod
-    def build_criterion(cls, args, task):
-        return cls(
-            task,
-            args.silence_token,
-            args.asg_transitions_init,
-            args.max_replabel,
-            args.linseg_updates,
-            args.hide_linseg_messages,
-        )
-
-    def linseg_step(self):
-        if not self.training:
-            return False
-        if self.linseg_progress.item() < self.linseg_maximum:
-            if self.linseg_message_state == "start":
-                print("| using LinSeg to initialize ASG")
-                self.linseg_message_state = "finish"
-            self.linseg_progress.add_(1)
-            return True
-        elif self.linseg_message_state == "finish":
-            print("| finished LinSeg initialization")
-            self.linseg_message_state = "none"
-        return False
-
-    def replace_eos_with_silence(self, tgt):
-        if tgt[-1] != self.eos:
-            return tgt
-        elif self.silence is None or (len(tgt) > 1 and tgt[-2] == self.silence):
-            return tgt[:-1]
-        else:
-            return tgt[:-1] + [self.silence]
-
-    def forward(self, model, sample, reduce=True):
-        """Compute the loss for the given sample.
-
-        Returns a tuple with three elements:
-        1) the loss
-        2) the sample size, which is used as the denominator for the gradient
-        3) logging outputs to display while training
-        """
-
-        net_output = model(**sample["net_input"])
-        emissions = net_output["encoder_out"].transpose(0, 1).contiguous()
-        B = emissions.size(0)
-        T = emissions.size(1)
-        device = emissions.device
-
-        target = torch.IntTensor(B, T)
-        target_size = torch.IntTensor(B)
-        using_linseg = self.linseg_step()
-
-        for b in range(B):
-            initial_target_size = sample["target_lengths"][b].item()
-            if initial_target_size == 0:
-                raise ValueError("target size cannot be zero")
-
-            tgt = sample["target"][b, :initial_target_size].tolist()
-            tgt = self.replace_eos_with_silence(tgt)
-            tgt = pack_replabels(tgt, self.tgt_dict, self.max_replabel)
-            tgt = tgt[:T]
-
-            if using_linseg:
-                tgt = [tgt[t * len(tgt) // T] for t in range(T)]
-
-            target[b][: len(tgt)] = torch.IntTensor(tgt)
-            target_size[b] = len(tgt)
-
-        loss = self.asg.forward(emissions, target.to(device), target_size.to(device))
-
-        if reduce:
-            loss = torch.sum(loss)
-
-        sample_size = (
-            sample["target"].size(0) if self.args.sentence_avg else sample["ntokens"]
-        )
-        logging_output = {
-            "loss": utils.item(loss.data) if reduce else loss.data,
-            "ntokens": sample["ntokens"],
-            "nsentences": sample["target"].size(0),
-            "sample_size": sample_size,
-        }
-        return loss, sample_size, logging_output
-
-    @staticmethod
-    def aggregate_logging_outputs(logging_outputs):
-        """Aggregate logging outputs from data parallel training."""
-        loss_sum = sum(log.get("loss", 0) for log in logging_outputs)
-        ntokens = sum(log.get("ntokens", 0) for log in logging_outputs)
-        nsentences = sum(log.get("nsentences", 0) for log in logging_outputs)
-        sample_size = sum(log.get("sample_size", 0) for log in logging_outputs)
-        agg_output = {
-            "loss": loss_sum / nsentences,
-            "ntokens": ntokens,
-            "nsentences": nsentences,
-            "sample_size": sample_size,
-        }
-        return agg_output

audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/criterions/__init__.py

@@ -1,16 +0,0 @@
-import importlib
-import os
-
-# ASG loss requires flashlight bindings
-files_to_skip = set()
-try:
-    import flashlight.lib.sequence.criterion
-except ImportError:
-    files_to_skip.add("ASG_loss.py")
-
-for file in sorted(os.listdir(os.path.dirname(__file__))):
-    if file.endswith(".py") and not file.startswith("_") and file not in files_to_skip:
-        criterion_name = file[: file.find(".py")]
-        importlib.import_module(
-            "examples.speech_recognition.criterions." + criterion_name
-        )

audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/criterions/cross_entropy_acc.py

@@ -1,130 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-from __future__ import absolute_import, division, print_function, unicode_literals
-
-import logging
-import math
-
-import torch
-import torch.nn.functional as F
-from fairseq import utils
-from fairseq.criterions import FairseqCriterion, register_criterion
-
-
-@register_criterion("cross_entropy_acc")
-class CrossEntropyWithAccCriterion(FairseqCriterion):
-    def __init__(self, task, sentence_avg):
-        super().__init__(task)
-        self.sentence_avg = sentence_avg
-
-    def compute_loss(self, model, net_output, target, reduction, log_probs):
-        # N, T -> N * T
-        target = target.view(-1)
-        lprobs = model.get_normalized_probs(net_output, log_probs=log_probs)
-        if not hasattr(lprobs, "batch_first"):
-            logging.warning(
-                "ERROR: we need to know whether "
-                "batch first for the net output; "
-                "you need to set batch_first attribute for the return value of "
-                "model.get_normalized_probs. Now, we assume this is true, but "
-                "in the future, we will raise exception instead. "
-            )
-        batch_first = getattr(lprobs, "batch_first", True)
-        if not batch_first:
-            lprobs = lprobs.transpose(0, 1)
-
-        # N, T, D -> N * T, D
-        lprobs = lprobs.view(-1, lprobs.size(-1))
-        loss = F.nll_loss(
-            lprobs, target, ignore_index=self.padding_idx, reduction=reduction
-        )
-        return lprobs, loss
-
-    def get_logging_output(self, sample, target, lprobs, loss):
-        target = target.view(-1)
-        mask = target != self.padding_idx
-        correct = torch.sum(
-            lprobs.argmax(1).masked_select(mask) == target.masked_select(mask)
-        )
-        total = torch.sum(mask)
-        sample_size = (
-            sample["target"].size(0) if self.sentence_avg else sample["ntokens"]
-        )
-
-        logging_output = {
-            "loss": utils.item(loss.data),  # * sample['ntokens'],
-            "ntokens": sample["ntokens"],
-            "nsentences": sample["target"].size(0),
-            "sample_size": sample_size,
-            "correct": utils.item(correct.data),
-            "total": utils.item(total.data),
-            "nframes": torch.sum(sample["net_input"]["src_lengths"]).item(),
-        }
-
-        return sample_size, logging_output
-
-    def forward(self, model, sample, reduction="sum", log_probs=True):
-        """Computes the cross entropy with accuracy metric for the given sample.
-
-        This is similar to CrossEntropyCriterion in fairseq, but also
-        computes accuracy metrics as part of logging
-
-        Args:
-            logprobs (Torch.tensor) of shape N, T, D i.e.
-                batchsize, timesteps, dimensions
-            targets (Torch.tensor) of shape N, T  i.e batchsize, timesteps
-
-        Returns:
-        tuple: With three elements:
-            1) the loss
-            2) the sample size, which is used as the denominator for the gradient
-            3) logging outputs to display while training
-
-        TODO:
-            * Currently this Criterion will only work with LSTMEncoderModels or
-            FairseqModels which have decoder, or Models which return TorchTensor
-            as net_output.
-            We need to make a change to support all FairseqEncoder models.
-        """
-        net_output = model(**sample["net_input"])
-        target = model.get_targets(sample, net_output)
-        lprobs, loss = self.compute_loss(
-            model, net_output, target, reduction, log_probs
-        )
-        sample_size, logging_output = self.get_logging_output(
-            sample, target, lprobs, loss
-        )
-        return loss, sample_size, logging_output
-
-    @staticmethod
-    def aggregate_logging_outputs(logging_outputs):
-        """Aggregate logging outputs from data parallel training."""
-        correct_sum = sum(log.get("correct", 0) for log in logging_outputs)
-        total_sum = sum(log.get("total", 0) for log in logging_outputs)
-        loss_sum = sum(log.get("loss", 0) for log in logging_outputs)
-        ntokens = sum(log.get("ntokens", 0) for log in logging_outputs)
-        nsentences = sum(log.get("nsentences", 0) for log in logging_outputs)
-        sample_size = sum(log.get("sample_size", 0) for log in logging_outputs)
-        nframes = sum(log.get("nframes", 0) for log in logging_outputs)
-        agg_output = {
-            "loss": loss_sum / sample_size / math.log(2) if sample_size > 0 else 0.0,
-            # if args.sentence_avg, then sample_size is nsentences, then loss
-            # is per-sentence loss; else sample_size is ntokens, the loss
-            # becomes per-output token loss
-            "ntokens": ntokens,
-            "nsentences": nsentences,
-            "nframes": nframes,
-            "sample_size": sample_size,
-            "acc": correct_sum * 100.0 / total_sum if total_sum > 0 else 0.0,
-            "correct": correct_sum,
-            "total": total_sum,
-            # total is the number of validate tokens
-        }
-        if sample_size != ntokens:
-            agg_output["nll_loss"] = loss_sum / ntokens / math.log(2)
-        # loss: per output token loss
-        # nll_loss: per sentence loss
-        return agg_output

audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/data/__init__.py

@@ -1,10 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-from .asr_dataset import AsrDataset
-
-__all__ = [
-    "AsrDataset",
-]

audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/data/asr_dataset.py

@@ -1,122 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-import os
-
-import numpy as np
-from fairseq.data import FairseqDataset
-
-from . import data_utils
-from .collaters import Seq2SeqCollater
-
-
-class AsrDataset(FairseqDataset):
-    """
-    A dataset representing speech and corresponding transcription.
-
-    Args:
-        aud_paths: (List[str]): A list of str with paths to audio files.
-        aud_durations_ms (List[int]): A list of int containing the durations of
-            audio files.
-        tgt (List[torch.LongTensor]): A list of LongTensors containing the indices
-            of target transcriptions.
-        tgt_dict (~fairseq.data.Dictionary): target vocabulary.
-        ids (List[str]): A list of utterance IDs.
-        speakers (List[str]): A list of speakers corresponding to utterances.
-        num_mel_bins (int): Number of triangular mel-frequency bins (default: 80)
-        frame_length (float): Frame length in milliseconds (default: 25.0)
-        frame_shift (float): Frame shift in milliseconds (default: 10.0)
-    """
-
-    def __init__(
-        self,
-        aud_paths,
-        aud_durations_ms,
-        tgt,
-        tgt_dict,
-        ids,
-        speakers,
-        num_mel_bins=80,
-        frame_length=25.0,
-        frame_shift=10.0,
-    ):
-        assert frame_length > 0
-        assert frame_shift > 0
-        assert all(x > frame_length for x in aud_durations_ms)
-        self.frame_sizes = [
-            int(1 + (d - frame_length) / frame_shift) for d in aud_durations_ms
-        ]
-
-        assert len(aud_paths) > 0
-        assert len(aud_paths) == len(aud_durations_ms)
-        assert len(aud_paths) == len(tgt)
-        assert len(aud_paths) == len(ids)
-        assert len(aud_paths) == len(speakers)
-        self.aud_paths = aud_paths
-        self.tgt_dict = tgt_dict
-        self.tgt = tgt
-        self.ids = ids
-        self.speakers = speakers
-        self.num_mel_bins = num_mel_bins
-        self.frame_length = frame_length
-        self.frame_shift = frame_shift
-
-        self.s2s_collater = Seq2SeqCollater(
-            0,
-            1,
-            pad_index=self.tgt_dict.pad(),
-            eos_index=self.tgt_dict.eos(),
-            move_eos_to_beginning=True,
-        )
-
-    def __getitem__(self, index):
-        import torchaudio
-        import torchaudio.compliance.kaldi as kaldi
-
-        tgt_item = self.tgt[index] if self.tgt is not None else None
-
-        path = self.aud_paths[index]
-        if not os.path.exists(path):
-            raise FileNotFoundError("Audio file not found: {}".format(path))
-        sound, sample_rate = torchaudio.load_wav(path)
-        output = kaldi.fbank(
-            sound,
-            num_mel_bins=self.num_mel_bins,
-            frame_length=self.frame_length,
-            frame_shift=self.frame_shift,
-        )
-        output_cmvn = data_utils.apply_mv_norm(output)
-
-        return {"id": index, "data": [output_cmvn.detach(), tgt_item]}
-
-    def __len__(self):
-        return len(self.aud_paths)
-
-    def collater(self, samples):
-        """Merge a list of samples to form a mini-batch.
-
-        Args:
-            samples (List[int]): sample indices to collate
-
-        Returns:
-            dict: a mini-batch suitable for forwarding with a Model
-        """
-        return self.s2s_collater.collate(samples)
-
-    def num_tokens(self, index):
-        return self.frame_sizes[index]
-
-    def size(self, index):
-        """Return an example's size as a float or tuple. This value is used when
-        filtering a dataset with ``--max-positions``."""
-        return (
-            self.frame_sizes[index],
-            len(self.tgt[index]) if self.tgt is not None else 0,
-        )
-
-    def ordered_indices(self):
-        """Return an ordered list of indices. Batches will be constructed based
-        on this order."""
-        return np.arange(len(self))

audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/data/collaters.py

@@ -1,131 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-"""
-    This module contains collection of classes which implement
-    collate functionalities for various tasks.
-
-    Collaters should know what data to expect for each sample
-    and they should pack / collate them into batches
-"""
-
-
-from __future__ import absolute_import, division, print_function, unicode_literals
-
-import numpy as np
-import torch
-from fairseq.data import data_utils as fairseq_data_utils
-
-
-class Seq2SeqCollater(object):
-    """
-    Implements collate function mainly for seq2seq tasks
-    This expects each sample to contain feature (src_tokens) and
-    targets.
-    This collator is also used for aligned training task.
-    """
-
-    def __init__(
-        self,
-        feature_index=0,
-        label_index=1,
-        pad_index=1,
-        eos_index=2,
-        move_eos_to_beginning=True,
-    ):
-        self.feature_index = feature_index
-        self.label_index = label_index
-        self.pad_index = pad_index
-        self.eos_index = eos_index
-        self.move_eos_to_beginning = move_eos_to_beginning
-
-    def _collate_frames(self, frames):
-        """Convert a list of 2d frames into a padded 3d tensor
-        Args:
-            frames (list): list of 2d frames of size L[i]*f_dim. Where L[i] is
-                length of i-th frame and f_dim is static dimension of features
-        Returns:
-            3d tensor of size len(frames)*len_max*f_dim where len_max is max of L[i]
-        """
-        len_max = max(frame.size(0) for frame in frames)
-        f_dim = frames[0].size(1)
-        res = frames[0].new(len(frames), len_max, f_dim).fill_(0.0)
-
-        for i, v in enumerate(frames):
-            res[i, : v.size(0)] = v
-
-        return res
-
-    def collate(self, samples):
-        """
-        utility function to collate samples into batch for speech recognition.
-        """
-        if len(samples) == 0:
-            return {}
-
-        # parse samples into torch tensors
-        parsed_samples = []
-        for s in samples:
-            # skip invalid samples
-            if s["data"][self.feature_index] is None:
-                continue
-            source = s["data"][self.feature_index]
-            if isinstance(source, (np.ndarray, np.generic)):
-                source = torch.from_numpy(source)
-            target = s["data"][self.label_index]
-            if isinstance(target, (np.ndarray, np.generic)):
-                target = torch.from_numpy(target).long()
-            elif isinstance(target, list):
-                target = torch.LongTensor(target)
-
-            parsed_sample = {"id": s["id"], "source": source, "target": target}
-            parsed_samples.append(parsed_sample)
-        samples = parsed_samples
-
-        id = torch.LongTensor([s["id"] for s in samples])
-        frames = self._collate_frames([s["source"] for s in samples])
-        # sort samples by descending number of frames
-        frames_lengths = torch.LongTensor([s["source"].size(0) for s in samples])
-        frames_lengths, sort_order = frames_lengths.sort(descending=True)
-        id = id.index_select(0, sort_order)
-        frames = frames.index_select(0, sort_order)
-
-        target = None
-        target_lengths = None
-        prev_output_tokens = None
-        if samples[0].get("target", None) is not None:
-            ntokens = sum(len(s["target"]) for s in samples)
-            target = fairseq_data_utils.collate_tokens(
-                [s["target"] for s in samples],
-                self.pad_index,
-                self.eos_index,
-                left_pad=False,
-                move_eos_to_beginning=False,
-            )
-            target = target.index_select(0, sort_order)
-            target_lengths = torch.LongTensor(
-                [s["target"].size(0) for s in samples]
-            ).index_select(0, sort_order)
-            prev_output_tokens = fairseq_data_utils.collate_tokens(
-                [s["target"] for s in samples],
-                self.pad_index,
-                self.eos_index,
-                left_pad=False,
-                move_eos_to_beginning=self.move_eos_to_beginning,
-            )
-            prev_output_tokens = prev_output_tokens.index_select(0, sort_order)
-        else:
-            ntokens = sum(len(s["source"]) for s in samples)
-
-        batch = {
-            "id": id,
-            "ntokens": ntokens,
-            "net_input": {"src_tokens": frames, "src_lengths": frames_lengths},
-            "target": target,
-            "target_lengths": target_lengths,
-            "nsentences": len(samples),
-        }
-        if prev_output_tokens is not None:
-            batch["net_input"]["prev_output_tokens"] = prev_output_tokens
-        return batch

audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/data/data_utils.py

@@ -1,100 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-import torch
-
-
-def calc_mean_invstddev(feature):
-    if len(feature.size()) != 2:
-        raise ValueError("We expect the input feature to be 2-D tensor")
-    mean = feature.mean(0)
-    var = feature.var(0)
-    # avoid division by ~zero
-    eps = 1e-8
-    if (var < eps).any():
-        return mean, 1.0 / (torch.sqrt(var) + eps)
-    return mean, 1.0 / torch.sqrt(var)
-
-
-def apply_mv_norm(features):
-    # If there is less than 2 spectrograms, the variance cannot be computed (is NaN)
-    # and normalization is not possible, so return the item as it is
-    if features.size(0) < 2:
-        return features
-    mean, invstddev = calc_mean_invstddev(features)
-    res = (features - mean) * invstddev
-    return res
-
-
-def lengths_to_encoder_padding_mask(lengths, batch_first=False):
-    """
-    convert lengths (a 1-D Long/Int tensor) to 2-D binary tensor
-
-    Args:
-        lengths: a (B, )-shaped tensor
-
-    Return:
-        max_length: maximum length of B sequences
-        encoder_padding_mask: a (max_length, B) binary mask, where
-        [t, b] = 0 for t < lengths[b] and 1 otherwise
-
-    TODO:
-        kernelize this function if benchmarking shows this function is slow
-    """
-    max_lengths = torch.max(lengths).item()
-    bsz = lengths.size(0)
-    encoder_padding_mask = torch.arange(
-        max_lengths
-    ).to(  # a (T, ) tensor with [0, ..., T-1]
-        lengths.device
-    ).view(  # move to the right device
-        1, max_lengths
-    ).expand(  # reshape to (1, T)-shaped tensor
-        bsz, -1
-    ) >= lengths.view(  # expand to (B, T)-shaped tensor
-        bsz, 1
-    ).expand(
-        -1, max_lengths
-    )
-    if not batch_first:
-        return encoder_padding_mask.t(), max_lengths
-    else:
-        return encoder_padding_mask, max_lengths
-
-
-def encoder_padding_mask_to_lengths(
-    encoder_padding_mask, max_lengths, batch_size, device
-):
-    """
-    convert encoder_padding_mask (2-D binary tensor) to a 1-D tensor
-
-    Conventionally, encoder output contains a encoder_padding_mask, which is
-    a 2-D mask in a shape (T, B), whose (t, b) element indicate whether
-    encoder_out[t, b] is a valid output (=0) or not (=1). Occasionally, we
-    need to convert this mask tensor to a 1-D tensor in shape (B, ), where
-    [b] denotes the valid length of b-th sequence
-
-    Args:
-        encoder_padding_mask: a (T, B)-shaped binary tensor or None; if None,
-        indicating all are valid
-    Return:
-        seq_lengths: a (B,)-shaped tensor, where its (b, )-th element is the
-        number of valid elements of b-th sequence
-
-        max_lengths: maximum length of all sequence, if encoder_padding_mask is
-        not None, max_lengths must equal to encoder_padding_mask.size(0)
-
-        batch_size: batch size; if encoder_padding_mask is
-        not None, max_lengths must equal to encoder_padding_mask.size(1)
-
-        device: which device to put the result on
-    """
-    if encoder_padding_mask is None:
-        return torch.Tensor([max_lengths] * batch_size).to(torch.int32).to(device)
-
-    assert encoder_padding_mask.size(0) == max_lengths, "max_lengths does not match"
-    assert encoder_padding_mask.size(1) == batch_size, "batch_size does not match"
-
-    return max_lengths - torch.sum(encoder_padding_mask, dim=0)

audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/data/replabels.py

@@ -1,70 +0,0 @@
-#!/usr/bin/env python3
-
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-"""
-Replabel transforms for use with flashlight's ASG criterion.
-"""
-
-
-def replabel_symbol(i):
-    """
-    Replabel symbols used in flashlight, currently just "1", "2", ...
-    This prevents training with numeral tokens, so this might change in the future
-    """
-    return str(i)
-
-
-def pack_replabels(tokens, dictionary, max_reps):
-    """
-    Pack a token sequence so that repeated symbols are replaced by replabels
-    """
-    if len(tokens) == 0 or max_reps <= 0:
-        return tokens
-
-    replabel_value_to_idx = [0] * (max_reps + 1)
-    for i in range(1, max_reps + 1):
-        replabel_value_to_idx[i] = dictionary.index(replabel_symbol(i))
-
-    result = []
-    prev_token = -1
-    num_reps = 0
-    for token in tokens:
-        if token == prev_token and num_reps < max_reps:
-            num_reps += 1
-        else:
-            if num_reps > 0:
-                result.append(replabel_value_to_idx[num_reps])
-                num_reps = 0
-            result.append(token)
-            prev_token = token
-    if num_reps > 0:
-        result.append(replabel_value_to_idx[num_reps])
-    return result
-
-
-def unpack_replabels(tokens, dictionary, max_reps):
-    """
-    Unpack a token sequence so that replabels are replaced by repeated symbols
-    """
-    if len(tokens) == 0 or max_reps <= 0:
-        return tokens
-
-    replabel_idx_to_value = {}
-    for i in range(1, max_reps + 1):
-        replabel_idx_to_value[dictionary.index(replabel_symbol(i))] = i
-
-    result = []
-    prev_token = -1
-    for token in tokens:
-        try:
-            for _ in range(replabel_idx_to_value[token]):
-                result.append(prev_token)
-            prev_token = -1
-        except KeyError:
-            result.append(token)
-            prev_token = token
-    return result

audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/datasets/asr_prep_json.py

@@ -1,124 +0,0 @@
-#!/usr/bin/env python3
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-from __future__ import absolute_import, division, print_function, unicode_literals
-
-import argparse
-import concurrent.futures
-import json
-import multiprocessing
-import os
-from collections import namedtuple
-from itertools import chain
-
-import sentencepiece as spm
-from fairseq.data import Dictionary
-
-MILLISECONDS_TO_SECONDS = 0.001
-
-
-def process_sample(aud_path, lable, utt_id, sp, tgt_dict):
-    import torchaudio
-
-    input = {}
-    output = {}
-    si, ei = torchaudio.info(aud_path)
-    input["length_ms"] = int(
-        si.length / si.channels / si.rate / MILLISECONDS_TO_SECONDS
-    )
-    input["path"] = aud_path
-
-    token = " ".join(sp.EncodeAsPieces(lable))
-    ids = tgt_dict.encode_line(token, append_eos=False)
-    output["text"] = lable
-    output["token"] = token
-    output["tokenid"] = ", ".join(map(str, [t.tolist() for t in ids]))
-    return {utt_id: {"input": input, "output": output}}
-
-
-def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "--audio-dirs",
-        nargs="+",
-        default=["-"],
-        required=True,
-        help="input directories with audio files",
-    )
-    parser.add_argument(
-        "--labels",
-        required=True,
-        help="aggregated input labels with format <ID LABEL> per line",
-        type=argparse.FileType("r", encoding="UTF-8"),
-    )
-    parser.add_argument(
-        "--spm-model",
-        required=True,
-        help="sentencepiece model to use for encoding",
-        type=argparse.FileType("r", encoding="UTF-8"),
-    )
-    parser.add_argument(
-        "--dictionary",
-        required=True,
-        help="file to load fairseq dictionary from",
-        type=argparse.FileType("r", encoding="UTF-8"),
-    )
-    parser.add_argument("--audio-format", choices=["flac", "wav"], default="wav")
-    parser.add_argument(
-        "--output",
-        required=True,
-        type=argparse.FileType("w"),
-        help="path to save json output",
-    )
-    args = parser.parse_args()
-
-    sp = spm.SentencePieceProcessor()
-    sp.Load(args.spm_model.name)
-
-    tgt_dict = Dictionary.load(args.dictionary)
-
-    labels = {}
-    for line in args.labels:
-        (utt_id, label) = line.split(" ", 1)
-        labels[utt_id] = label
-    if len(labels) == 0:
-        raise Exception("No labels found in ", args.labels_path)
-
-    Sample = namedtuple("Sample", "aud_path utt_id")
-    samples = []
-    for path, _, files in chain.from_iterable(
-        os.walk(path) for path in args.audio_dirs
-    ):
-        for f in files:
-            if f.endswith(args.audio_format):
-                if len(os.path.splitext(f)) != 2:
-                    raise Exception("Expect <utt_id.extension> file name. Got: ", f)
-                utt_id = os.path.splitext(f)[0]
-                if utt_id not in labels:
-                    continue
-                samples.append(Sample(os.path.join(path, f), utt_id))
-
-    utts = {}
-    num_cpu = multiprocessing.cpu_count()
-    with concurrent.futures.ThreadPoolExecutor(max_workers=num_cpu) as executor:
-        future_to_sample = {
-            executor.submit(
-                process_sample, s.aud_path, labels[s.utt_id], s.utt_id, sp, tgt_dict
-            ): s
-            for s in samples
-        }
-        for future in concurrent.futures.as_completed(future_to_sample):
-            try:
-                data = future.result()
-            except Exception as exc:
-                print("generated an exception: ", exc)
-            else:
-                utts.update(data)
-    json.dump({"utts": utts}, args.output, indent=4)
-
-
-if __name__ == "__main__":
-    main()

audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/datasets/prepare-librispeech.sh

@@ -1,88 +0,0 @@
-#!/usr/bin/env bash
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-# Prepare librispeech dataset
-
-base_url=www.openslr.org/resources/12
-train_dir=train_960
-
-if [ "$#" -ne 2 ]; then
-  echo "Usage: $0 <download_dir> <out_dir>"
-  echo "e.g.: $0 /tmp/librispeech_raw/ ~/data/librispeech_final"
-  exit 1
-fi
-
-download_dir=${1%/}
-out_dir=${2%/}
-
-fairseq_root=~/fairseq-py/
-mkdir -p ${out_dir}
-cd ${out_dir} || exit
-
-nbpe=5000
-bpemode=unigram
-
-if [ ! -d "$fairseq_root" ]; then
-    echo "$0: Please set correct fairseq_root"
-    exit 1
-fi
-
-echo "Data Download"
-for part in dev-clean test-clean dev-other test-other train-clean-100 train-clean-360 train-other-500; do
-    url=$base_url/$part.tar.gz
-    if ! wget -P $download_dir $url; then
-        echo "$0: wget failed for $url"
-        exit 1
-    fi
-    if ! tar -C $download_dir -xvzf $download_dir/$part.tar.gz; then
-        echo "$0: error un-tarring archive $download_dir/$part.tar.gz"
-        exit 1
-    fi
-done
-
-echo "Merge all train packs into one"
-mkdir -p ${download_dir}/LibriSpeech/${train_dir}/
-for part in train-clean-100 train-clean-360 train-other-500; do
-    mv ${download_dir}/LibriSpeech/${part}/* $download_dir/LibriSpeech/${train_dir}/
-done
-echo "Merge train text"
-find ${download_dir}/LibriSpeech/${train_dir}/ -name '*.txt' -exec cat {} \; >> ${download_dir}/LibriSpeech/${train_dir}/text
-
-# Use combined dev-clean and dev-other as validation set
-find ${download_dir}/LibriSpeech/dev-clean/ ${download_dir}/LibriSpeech/dev-other/ -name '*.txt' -exec cat {} \; >> ${download_dir}/LibriSpeech/valid_text
-find ${download_dir}/LibriSpeech/test-clean/ -name '*.txt' -exec cat {} \; >> ${download_dir}/LibriSpeech/test-clean/text
-find ${download_dir}/LibriSpeech/test-other/ -name '*.txt' -exec cat {} \; >> ${download_dir}/LibriSpeech/test-other/text
-
-
-dict=data/lang_char/${train_dir}_${bpemode}${nbpe}_units.txt
-encoded=data/lang_char/${train_dir}_${bpemode}${nbpe}_encoded.txt
-fairseq_dict=data/lang_char/${train_dir}_${bpemode}${nbpe}_fairseq_dict.txt
-bpemodel=data/lang_char/${train_dir}_${bpemode}${nbpe}
-echo "dictionary: ${dict}"
-echo "Dictionary preparation"
-mkdir -p data/lang_char/
-echo "<unk> 3" > ${dict}
-echo "</s> 2" >> ${dict}
-echo "<pad> 1" >> ${dict}
-cut -f 2- -d" " ${download_dir}/LibriSpeech/${train_dir}/text > data/lang_char/input.txt
-spm_train --input=data/lang_char/input.txt --vocab_size=${nbpe} --model_type=${bpemode} --model_prefix=${bpemodel} --input_sentence_size=100000000 --unk_id=3 --eos_id=2 --pad_id=1 --bos_id=-1 --character_coverage=1
-spm_encode --model=${bpemodel}.model --output_format=piece < data/lang_char/input.txt > ${encoded}
-cat ${encoded} | tr ' ' '\n' | sort | uniq | awk '{print $0 " " NR+3}' >> ${dict}
-cat ${encoded} | tr ' ' '\n' | sort | uniq -c | awk '{print $2 " " $1}' > ${fairseq_dict}
-wc -l ${dict}
-
-echo "Prepare train and test jsons"
-for part in train_960 test-other test-clean; do
-    python ${fairseq_root}/examples/speech_recognition/datasets/asr_prep_json.py --audio-dirs ${download_dir}/LibriSpeech/${part} --labels ${download_dir}/LibriSpeech/${part}/text --spm-model ${bpemodel}.model --audio-format flac --dictionary ${fairseq_dict} --output ${part}.json
-done
-# fairseq expects to find train.json and valid.json during training
-mv train_960.json train.json
-
-echo "Prepare valid json"
-python ${fairseq_root}/examples/speech_recognition/datasets/asr_prep_json.py --audio-dirs ${download_dir}/LibriSpeech/dev-clean ${download_dir}/LibriSpeech/dev-other --labels ${download_dir}/LibriSpeech/valid_text --spm-model ${bpemodel}.model --audio-format flac --dictionary ${fairseq_dict} --output valid.json
-
-cp ${fairseq_dict} ./dict.txt
-cp ${bpemodel}.model ./spm.model

audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/infer.py

@@ -1,435 +0,0 @@
-#!/usr/bin/env python3 -u
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-"""
-Run inference for pre-processed data with a trained model.
-"""
-
-import ast
-import logging
-import math
-import os
-import sys
-
-import editdistance
-import numpy as np
-import torch
-from fairseq import checkpoint_utils, options, progress_bar, tasks, utils
-from fairseq.data.data_utils import post_process
-from fairseq.logging.meters import StopwatchMeter, TimeMeter
-
-logging.basicConfig()
-logging.root.setLevel(logging.INFO)
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
-
-
-def add_asr_eval_argument(parser):
-    parser.add_argument("--kspmodel", default=None, help="sentence piece model")
-    parser.add_argument(
-        "--wfstlm", default=None, help="wfstlm on dictonary output units"
-    )
-    parser.add_argument(
-        "--rnnt_decoding_type",
-        default="greedy",
-        help="wfstlm on dictonary\
-output units",
-    )
-    try:
-        parser.add_argument(
-            "--lm-weight",
-            "--lm_weight",
-            type=float,
-            default=0.2,
-            help="weight for lm while interpolating with neural score",
-        )
-    except:
-        pass
-    parser.add_argument(
-        "--rnnt_len_penalty", default=-0.5, help="rnnt length penalty on word level"
-    )
-    parser.add_argument(
-        "--w2l-decoder",
-        choices=["viterbi", "kenlm", "fairseqlm"],
-        help="use a w2l decoder",
-    )
-    parser.add_argument("--lexicon", help="lexicon for w2l decoder")
-    parser.add_argument("--unit-lm", action="store_true", help="if using a unit lm")
-    parser.add_argument("--kenlm-model", "--lm-model", help="lm model for w2l decoder")
-    parser.add_argument("--beam-threshold", type=float, default=25.0)
-    parser.add_argument("--beam-size-token", type=float, default=100)
-    parser.add_argument("--word-score", type=float, default=1.0)
-    parser.add_argument("--unk-weight", type=float, default=-math.inf)
-    parser.add_argument("--sil-weight", type=float, default=0.0)
-    parser.add_argument(
-        "--dump-emissions",
-        type=str,
-        default=None,
-        help="if present, dumps emissions into this file and exits",
-    )
-    parser.add_argument(
-        "--dump-features",
-        type=str,
-        default=None,
-        help="if present, dumps features into this file and exits",
-    )
-    parser.add_argument(
-        "--load-emissions",
-        type=str,
-        default=None,
-        help="if present, loads emissions from this file",
-    )
-    return parser
-
-
-def check_args(args):
-    # assert args.path is not None, "--path required for generation!"
-    # assert args.results_path is not None, "--results_path required for generation!"
-    assert (
-        not args.sampling or args.nbest == args.beam
-    ), "--sampling requires --nbest to be equal to --beam"
-    assert (
-        args.replace_unk is None or args.raw_text
-    ), "--replace-unk requires a raw text dataset (--raw-text)"
-
-
-def get_dataset_itr(args, task, models):
-    return task.get_batch_iterator(
-        dataset=task.dataset(args.gen_subset),
-        max_tokens=args.max_tokens,
-        max_sentences=args.batch_size,
-        max_positions=(sys.maxsize, sys.maxsize),
-        ignore_invalid_inputs=args.skip_invalid_size_inputs_valid_test,
-        required_batch_size_multiple=args.required_batch_size_multiple,
-        num_shards=args.num_shards,
-        shard_id=args.shard_id,
-        num_workers=args.num_workers,
-        data_buffer_size=args.data_buffer_size,
-    ).next_epoch_itr(shuffle=False)
-
-
-def process_predictions(
-    args, hypos, sp, tgt_dict, target_tokens, res_files, speaker, id
-):
-    for hypo in hypos[: min(len(hypos), args.nbest)]:
-        hyp_pieces = tgt_dict.string(hypo["tokens"].int().cpu())
-
-        if "words" in hypo:
-            hyp_words = " ".join(hypo["words"])
-        else:
-            hyp_words = post_process(hyp_pieces, args.post_process)
-
-        if res_files is not None:
-            print(
-                "{} ({}-{})".format(hyp_pieces, speaker, id),
-                file=res_files["hypo.units"],
-            )
-            print(
-                "{} ({}-{})".format(hyp_words, speaker, id),
-                file=res_files["hypo.words"],
-            )
-
-        tgt_pieces = tgt_dict.string(target_tokens)
-        tgt_words = post_process(tgt_pieces, args.post_process)
-
-        if res_files is not None:
-            print(
-                "{} ({}-{})".format(tgt_pieces, speaker, id),
-                file=res_files["ref.units"],
-            )
-            print(
-                "{} ({}-{})".format(tgt_words, speaker, id), file=res_files["ref.words"]
-            )
-
-        if not args.quiet:
-            logger.info("HYPO:" + hyp_words)
-            logger.info("TARGET:" + tgt_words)
-            logger.info("___________________")
-
-        hyp_words = hyp_words.split()
-        tgt_words = tgt_words.split()
-        return editdistance.eval(hyp_words, tgt_words), len(tgt_words)
-
-
-def prepare_result_files(args):
-    def get_res_file(file_prefix):
-        if args.num_shards > 1:
-            file_prefix = f"{args.shard_id}_{file_prefix}"
-        path = os.path.join(
-            args.results_path,
-            "{}-{}-{}.txt".format(
-                file_prefix, os.path.basename(args.path), args.gen_subset
-            ),
-        )
-        return open(path, "w", buffering=1)
-
-    if not args.results_path:
-        return None
-
-    return {
-        "hypo.words": get_res_file("hypo.word"),
-        "hypo.units": get_res_file("hypo.units"),
-        "ref.words": get_res_file("ref.word"),
-        "ref.units": get_res_file("ref.units"),
-    }
-
-
-def optimize_models(args, use_cuda, models):
-    """Optimize ensemble for generation"""
-    for model in models:
-        model.make_generation_fast_(
-            beamable_mm_beam_size=None if args.no_beamable_mm else args.beam,
-            need_attn=args.print_alignment,
-        )
-        if args.fp16:
-            model.half()
-        if use_cuda:
-            model.cuda()
-
-
-def apply_half(t):
-    if t.dtype is torch.float32:
-        return t.to(dtype=torch.half)
-    return t
-
-
-class ExistingEmissionsDecoder(object):
-    def __init__(self, decoder, emissions):
-        self.decoder = decoder
-        self.emissions = emissions
-
-    def generate(self, models, sample, **unused):
-        ids = sample["id"].cpu().numpy()
-        try:
-            emissions = np.stack(self.emissions[ids])
-        except:
-            print([x.shape for x in self.emissions[ids]])
-            raise Exception("invalid sizes")
-        emissions = torch.from_numpy(emissions)
-        return self.decoder.decode(emissions)
-
-
-def main(args, task=None, model_state=None):
-    check_args(args)
-
-    use_fp16 = args.fp16
-    if args.max_tokens is None and args.batch_size is None:
-        args.max_tokens = 4000000
-    logger.info(args)
-
-    use_cuda = torch.cuda.is_available() and not args.cpu
-
-    logger.info("| decoding with criterion {}".format(args.criterion))
-
-    task = tasks.setup_task(args)
-
-    # Load ensemble
-    if args.load_emissions:
-        models, criterions = [], []
-        task.load_dataset(args.gen_subset)
-    else:
-        logger.info("| loading model(s) from {}".format(args.path))
-        models, saved_cfg, task = checkpoint_utils.load_model_ensemble_and_task(
-            utils.split_paths(args.path, separator="\\"),
-            arg_overrides=ast.literal_eval(args.model_overrides),
-            task=task,
-            suffix=args.checkpoint_suffix,
-            strict=(args.checkpoint_shard_count == 1),
-            num_shards=args.checkpoint_shard_count,
-            state=model_state,
-        )
-        optimize_models(args, use_cuda, models)
-        task.load_dataset(args.gen_subset, task_cfg=saved_cfg.task)
-
-
-    # Set dictionary
-    tgt_dict = task.target_dictionary
-
-    logger.info(
-        "| {} {} {} examples".format(
-            args.data, args.gen_subset, len(task.dataset(args.gen_subset))
-        )
-    )
-
-    # hack to pass transitions to W2lDecoder
-    if args.criterion == "asg_loss":
-        raise NotImplementedError("asg_loss is currently not supported")
-        # trans = criterions[0].asg.trans.data
-        # args.asg_transitions = torch.flatten(trans).tolist()
-
-    # Load dataset (possibly sharded)
-    itr = get_dataset_itr(args, task, models)
-
-    # Initialize generator
-    gen_timer = StopwatchMeter()
-
-    def build_generator(args):
-        w2l_decoder = getattr(args, "w2l_decoder", None)
-        if w2l_decoder == "viterbi":
-            from examples.speech_recognition.w2l_decoder import W2lViterbiDecoder
-
-            return W2lViterbiDecoder(args, task.target_dictionary)
-        elif w2l_decoder == "kenlm":
-            from examples.speech_recognition.w2l_decoder import W2lKenLMDecoder
-
-            return W2lKenLMDecoder(args, task.target_dictionary)
-        elif w2l_decoder == "fairseqlm":
-            from examples.speech_recognition.w2l_decoder import W2lFairseqLMDecoder
-
-            return W2lFairseqLMDecoder(args, task.target_dictionary)
-        else:
-            print(
-                "only flashlight decoders with (viterbi, kenlm, fairseqlm) options are supported at the moment"
-            )
-
-    # please do not touch this unless you test both generate.py and infer.py with audio_pretraining task
-    generator = build_generator(args)
-
-    if args.load_emissions:
-        generator = ExistingEmissionsDecoder(
-            generator, np.load(args.load_emissions, allow_pickle=True)
-        )
-        logger.info("loaded emissions from " + args.load_emissions)
-
-    num_sentences = 0
-
-    if args.results_path is not None and not os.path.exists(args.results_path):
-        os.makedirs(args.results_path)
-
-    max_source_pos = (
-        utils.resolve_max_positions(
-            task.max_positions(), *[model.max_positions() for model in models]
-        ),
-    )
-
-    if max_source_pos is not None:
-        max_source_pos = max_source_pos[0]
-        if max_source_pos is not None:
-            max_source_pos = max_source_pos[0] - 1
-
-    if args.dump_emissions:
-        emissions = {}
-    if args.dump_features:
-        features = {}
-        models[0].bert.proj = None
-    else:
-        res_files = prepare_result_files(args)
-    errs_t = 0
-    lengths_t = 0
-    with progress_bar.build_progress_bar(args, itr) as t:
-        wps_meter = TimeMeter()
-        for sample in t:
-            sample = utils.move_to_cuda(sample) if use_cuda else sample
-            if use_fp16:
-                sample = utils.apply_to_sample(apply_half, sample)
-            if "net_input" not in sample:
-                continue
-
-            prefix_tokens = None
-            if args.prefix_size > 0:
-                prefix_tokens = sample["target"][:, : args.prefix_size]
-
-            gen_timer.start()
-            if args.dump_emissions:
-                with torch.no_grad():
-                    encoder_out = models[0](**sample["net_input"])
-                    emm = models[0].get_normalized_probs(encoder_out, log_probs=True)
-                    emm = emm.transpose(0, 1).cpu().numpy()
-                    for i, id in enumerate(sample["id"]):
-                        emissions[id.item()] = emm[i]
-                    continue
-            elif args.dump_features:
-                with torch.no_grad():
-                    encoder_out = models[0](**sample["net_input"])
-                    feat = encoder_out["encoder_out"].transpose(0, 1).cpu().numpy()
-                    for i, id in enumerate(sample["id"]):
-                        padding = (
-                            encoder_out["encoder_padding_mask"][i].cpu().numpy()
-                            if encoder_out["encoder_padding_mask"] is not None
-                            else None
-                        )
-                        features[id.item()] = (feat[i], padding)
-                    continue
-            hypos = task.inference_step(generator, models, sample, prefix_tokens)
-            num_generated_tokens = sum(len(h[0]["tokens"]) for h in hypos)
-            gen_timer.stop(num_generated_tokens)
-
-            for i, sample_id in enumerate(sample["id"].tolist()):
-                speaker = None
-                # id = task.dataset(args.gen_subset).ids[int(sample_id)]
-                id = sample_id
-                toks = (
-                    sample["target"][i, :]
-                    if "target_label" not in sample
-                    else sample["target_label"][i, :]
-                )
-                target_tokens = utils.strip_pad(toks, tgt_dict.pad()).int().cpu()
-                # Process top predictions
-                errs, length = process_predictions(
-                    args,
-                    hypos[i],
-                    None,
-                    tgt_dict,
-                    target_tokens,
-                    res_files,
-                    speaker,
-                    id,
-                )
-                errs_t += errs
-                lengths_t += length
-
-            wps_meter.update(num_generated_tokens)
-            t.log({"wps": round(wps_meter.avg)})
-            num_sentences += (
-                sample["nsentences"] if "nsentences" in sample else sample["id"].numel()
-            )
-
-    wer = None
-    if args.dump_emissions:
-        emm_arr = []
-        for i in range(len(emissions)):
-            emm_arr.append(emissions[i])
-        np.save(args.dump_emissions, emm_arr)
-        logger.info(f"saved {len(emissions)} emissions to {args.dump_emissions}")
-    elif args.dump_features:
-        feat_arr = []
-        for i in range(len(features)):
-            feat_arr.append(features[i])
-        np.save(args.dump_features, feat_arr)
-        logger.info(f"saved {len(features)} emissions to {args.dump_features}")
-    else:
-        if lengths_t > 0:
-            wer = errs_t * 100.0 / lengths_t
-            logger.info(f"WER: {wer}")
-
-        logger.info(
-            "| Processed {} sentences ({} tokens) in {:.1f}s ({:.2f}"
-            "sentences/s, {:.2f} tokens/s)".format(
-                num_sentences,
-                gen_timer.n,
-                gen_timer.sum,
-                num_sentences / gen_timer.sum,
-                1.0 / gen_timer.avg,
-            )
-        )
-        logger.info("| Generate {} with beam={}".format(args.gen_subset, args.beam))
-    return task, wer
-
-
-def make_parser():
-    parser = options.get_generation_parser()
-    parser = add_asr_eval_argument(parser)
-    return parser
-
-
-def cli_main():
-    parser = make_parser()
-    args = options.parse_args_and_arch(parser)
-    main(args)
-
-
-if __name__ == "__main__":
-    cli_main()

audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/kaldi/add-self-loop-simple.cc

@@ -1,94 +0,0 @@
-/*
- * Copyright (c) Facebook, Inc. and its affiliates.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE file in the root directory of this source tree.
- */
-
-#include <iostream>
-#include "fstext/fstext-lib.h" // @manual
-#include "util/common-utils.h" // @manual
-
-/*
- * This program is to modify a FST without self-loop by:
- *   for each incoming arc with non-eps input symbol, add a self-loop arc
- *   with that non-eps symbol as input and eps as output.
- *
- * This is to make sure the resultant FST can do deduplication for repeated
- * symbols, which is very common in acoustic model
- *
- */
-namespace {
-int32 AddSelfLoopsSimple(fst::StdVectorFst* fst) {
-  typedef fst::MutableArcIterator<fst::StdVectorFst> IterType;
-
-  int32 num_states_before = fst->NumStates();
-  fst::MakePrecedingInputSymbolsSame(false, fst);
-  int32 num_states_after = fst->NumStates();
-  KALDI_LOG << "There are " << num_states_before
-            << " states in the original FST; "
-            << " after MakePrecedingInputSymbolsSame, there are "
-            << num_states_after << " states " << std::endl;
-
-  auto weight_one = fst::StdArc::Weight::One();
-
-  int32 num_arc_added = 0;
-
-  fst::StdArc self_loop_arc;
-  self_loop_arc.weight = weight_one;
-
-  int32 num_states = fst->NumStates();
-  std::vector<std::set<int32>> incoming_non_eps_label_per_state(num_states);
-
-  for (int32 state = 0; state < num_states; state++) {
-    for (IterType aiter(fst, state); !aiter.Done(); aiter.Next()) {
-      fst::StdArc arc(aiter.Value());
-      if (arc.ilabel != 0) {
-        incoming_non_eps_label_per_state[arc.nextstate].insert(arc.ilabel);
-      }
-    }
-  }
-
-  for (int32 state = 0; state < num_states; state++) {
-    if (!incoming_non_eps_label_per_state[state].empty()) {
-      auto& ilabel_set = incoming_non_eps_label_per_state[state];
-      for (auto it = ilabel_set.begin(); it != ilabel_set.end(); it++) {
-        self_loop_arc.ilabel = *it;
-        self_loop_arc.olabel = 0;
-        self_loop_arc.nextstate = state;
-        fst->AddArc(state, self_loop_arc);
-        num_arc_added++;
-      }
-    }
-  }
-  return num_arc_added;
-}
-
-void print_usage() {
-  std::cout << "add-self-loop-simple usage:\n"
-               "\tadd-self-loop-simple <in-fst> <out-fst> \n";
-}
-} // namespace
-
-int main(int argc, char** argv) {
-  if (argc != 3) {
-    print_usage();
-    exit(1);
-  }
-
-  auto input = argv[1];
-  auto output = argv[2];
-
-  auto fst = fst::ReadFstKaldi(input);
-  auto num_states = fst->NumStates();
-  KALDI_LOG << "Loading FST from " << input << " with " << num_states
-            << " states." << std::endl;
-
-  int32 num_arc_added = AddSelfLoopsSimple(fst);
-  KALDI_LOG << "Adding " << num_arc_added << " self-loop arcs " << std::endl;
-
-  fst::WriteFstKaldi(*fst, std::string(output));
-  KALDI_LOG << "Writing FST to " << output << std::endl;
-
-  delete fst;
-}

audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/kaldi/config/kaldi_initializer.yaml

@@ -1,8 +0,0 @@
-# @package _group_
-
-data_dir: ???
-fst_dir: ???
-in_labels: ???
-kaldi_root: ???
-lm_arpa: ???
-blank_symbol: <s>

audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/kaldi/kaldi_decoder.py

@@ -1,243 +0,0 @@
-#!/usr/bin/env python3
-
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-import logging
-import os
-import warnings
-from concurrent.futures import ThreadPoolExecutor
-from dataclasses import dataclass
-from typing import Optional
-
-import torch
-from fairseq.dataclass import FairseqDataclass
-from omegaconf import MISSING
-
-from .kaldi_initializer import KaldiInitializerConfig, initalize_kaldi
-
-logger = logging.getLogger(__name__)
-
-
-@dataclass
-class KaldiDecoderConfig(FairseqDataclass):
-    hlg_graph_path: Optional[str] = None
-    output_dict: str = MISSING
-
-    kaldi_initializer_config: Optional[KaldiInitializerConfig] = None
-
-    acoustic_scale: float = 0.5
-    max_active: int = 10000
-    beam_delta: float = 0.5
-    hash_ratio: float = 2.0
-
-    is_lattice: bool = False
-    lattice_beam: float = 10.0
-    prune_interval: int = 25
-    determinize_lattice: bool = True
-    prune_scale: float = 0.1
-    max_mem: int = 0
-    phone_determinize: bool = True
-    word_determinize: bool = True
-    minimize: bool = True
-
-    num_threads: int = 1
-
-
-class KaldiDecoder(object):
-    def __init__(
-        self,
-        cfg: KaldiDecoderConfig,
-        beam: int,
-        nbest: int = 1,
-    ):
-        try:
-            from kaldi.asr import FasterRecognizer, LatticeFasterRecognizer
-            from kaldi.base import set_verbose_level
-            from kaldi.decoder import (
-                FasterDecoder,
-                FasterDecoderOptions,
-                LatticeFasterDecoder,
-                LatticeFasterDecoderOptions,
-            )
-            from kaldi.fstext import SymbolTable, read_fst_kaldi
-            from kaldi.lat.functions import DeterminizeLatticePhonePrunedOptions
-        except:
-            warnings.warn(
-                "pykaldi is required for this functionality. Please install from https://github.com/pykaldi/pykaldi"
-            )
-
-        # set_verbose_level(2)
-
-        self.acoustic_scale = cfg.acoustic_scale
-        self.nbest = nbest
-
-        if cfg.hlg_graph_path is None:
-            assert (
-                cfg.kaldi_initializer_config is not None
-            ), "Must provide hlg graph path or kaldi initializer config"
-            cfg.hlg_graph_path = initalize_kaldi(cfg.kaldi_initializer_config)
-
-        assert os.path.exists(cfg.hlg_graph_path), cfg.hlg_graph_path
-
-        if cfg.is_lattice:
-            self.dec_cls = LatticeFasterDecoder
-            opt_cls = LatticeFasterDecoderOptions
-            self.rec_cls = LatticeFasterRecognizer
-        else:
-            assert self.nbest == 1, "nbest > 1 requires lattice decoder"
-            self.dec_cls = FasterDecoder
-            opt_cls = FasterDecoderOptions
-            self.rec_cls = FasterRecognizer
-
-        self.decoder_options = opt_cls()
-        self.decoder_options.beam = beam
-        self.decoder_options.max_active = cfg.max_active
-        self.decoder_options.beam_delta = cfg.beam_delta
-        self.decoder_options.hash_ratio = cfg.hash_ratio
-
-        if cfg.is_lattice:
-            self.decoder_options.lattice_beam = cfg.lattice_beam
-            self.decoder_options.prune_interval = cfg.prune_interval
-            self.decoder_options.determinize_lattice = cfg.determinize_lattice
-            self.decoder_options.prune_scale = cfg.prune_scale
-            det_opts = DeterminizeLatticePhonePrunedOptions()
-            det_opts.max_mem = cfg.max_mem
-            det_opts.phone_determinize = cfg.phone_determinize
-            det_opts.word_determinize = cfg.word_determinize
-            det_opts.minimize = cfg.minimize
-            self.decoder_options.det_opts = det_opts
-
-        self.output_symbols = {}
-        with open(cfg.output_dict, "r") as f:
-            for line in f:
-                items = line.rstrip().split()
-                assert len(items) == 2
-                self.output_symbols[int(items[1])] = items[0]
-
-        logger.info(f"Loading FST from {cfg.hlg_graph_path}")
-        self.fst = read_fst_kaldi(cfg.hlg_graph_path)
-        self.symbol_table = SymbolTable.read_text(cfg.output_dict)
-
-        self.executor = ThreadPoolExecutor(max_workers=cfg.num_threads)
-
-    def generate(self, models, sample, **unused):
-        """Generate a batch of inferences."""
-        # model.forward normally channels prev_output_tokens into the decoder
-        # separately, but SequenceGenerator directly calls model.encoder
-        encoder_input = {
-            k: v for k, v in sample["net_input"].items() if k != "prev_output_tokens"
-        }
-        emissions, padding = self.get_emissions(models, encoder_input)
-        return self.decode(emissions, padding)
-
-    def get_emissions(self, models, encoder_input):
-        """Run encoder and normalize emissions"""
-        model = models[0]
-
-        all_encoder_out = [m(**encoder_input) for m in models]
-
-        if len(all_encoder_out) > 1:
-
-            if "encoder_out" in all_encoder_out[0]:
-                encoder_out = {
-                    "encoder_out": sum(e["encoder_out"] for e in all_encoder_out)
-                    / len(all_encoder_out),
-                    "encoder_padding_mask": all_encoder_out[0]["encoder_padding_mask"],
-                }
-                padding = encoder_out["encoder_padding_mask"]
-            else:
-                encoder_out = {
-                    "logits": sum(e["logits"] for e in all_encoder_out)
-                    / len(all_encoder_out),
-                    "padding_mask": all_encoder_out[0]["padding_mask"],
-                }
-                padding = encoder_out["padding_mask"]
-        else:
-            encoder_out = all_encoder_out[0]
-            padding = (
-                encoder_out["padding_mask"]
-                if "padding_mask" in encoder_out
-                else encoder_out["encoder_padding_mask"]
-            )
-
-        if hasattr(model, "get_logits"):
-            emissions = model.get_logits(encoder_out, normalize=True)
-        else:
-            emissions = model.get_normalized_probs(encoder_out, log_probs=True)
-
-        return (
-            emissions.cpu().float().transpose(0, 1),
-            padding.cpu() if padding is not None and padding.any() else None,
-        )
-
-    def decode_one(self, logits, padding):
-        from kaldi.matrix import Matrix
-
-        decoder = self.dec_cls(self.fst, self.decoder_options)
-        asr = self.rec_cls(
-            decoder, self.symbol_table, acoustic_scale=self.acoustic_scale
-        )
-
-        if padding is not None:
-            logits = logits[~padding]
-
-        mat = Matrix(logits.numpy())
-
-        out = asr.decode(mat)
-
-        if self.nbest > 1:
-            from kaldi.fstext import shortestpath
-            from kaldi.fstext.utils import (
-                convert_compact_lattice_to_lattice,
-                convert_lattice_to_std,
-                convert_nbest_to_list,
-                get_linear_symbol_sequence,
-            )
-
-            lat = out["lattice"]
-
-            sp = shortestpath(lat, nshortest=self.nbest)
-
-            sp = convert_compact_lattice_to_lattice(sp)
-            sp = convert_lattice_to_std(sp)
-            seq = convert_nbest_to_list(sp)
-
-            results = []
-            for s in seq:
-                _, o, w = get_linear_symbol_sequence(s)
-                words = list(self.output_symbols[z] for z in o)
-                results.append(
-                    {
-                        "tokens": words,
-                        "words": words,
-                        "score": w.value,
-                        "emissions": logits,
-                    }
-                )
-            return results
-        else:
-            words = out["text"].split()
-            return [
-                {
-                    "tokens": words,
-                    "words": words,
-                    "score": out["likelihood"],
-                    "emissions": logits,
-                }
-            ]
-
-    def decode(self, emissions, padding):
-        if padding is None:
-            padding = [None] * len(emissions)
-
-        ret = list(
-            map(
-                lambda e, p: self.executor.submit(self.decode_one, e, p),
-                emissions,
-                padding,
-            )
-        )
-        return ret

audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/kaldi/kaldi_initializer.py

@@ -1,698 +0,0 @@
-#!/usr/bin/env python3
-
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-import logging
-import os
-import os.path as osp
-import subprocess
-from dataclasses import dataclass
-from pathlib import Path
-from typing import Optional
-
-import hydra
-from fairseq.data.dictionary import Dictionary
-from fairseq.dataclass import FairseqDataclass
-from hydra.core.config_store import ConfigStore
-from omegaconf import MISSING, OmegaConf
-
-script_dir = Path(__file__).resolve().parent
-config_path = script_dir / "config"
-
-
-logger = logging.getLogger(__name__)
-
-
-@dataclass
-class KaldiInitializerConfig(FairseqDataclass):
-    data_dir: str = MISSING
-    fst_dir: Optional[str] = None
-    in_labels: str = MISSING
-    out_labels: Optional[str] = None
-    wav2letter_lexicon: Optional[str] = None
-    lm_arpa: str = MISSING
-    kaldi_root: str = MISSING
-    blank_symbol: str = "<s>"
-    silence_symbol: Optional[str] = None
-
-
-def create_units(fst_dir: Path, in_labels: str, vocab: Dictionary) -> Path:
-    in_units_file = fst_dir / f"kaldi_dict.{in_labels}.txt"
-    if not in_units_file.exists():
-
-        logger.info(f"Creating {in_units_file}")
-
-        with open(in_units_file, "w") as f:
-            print("<eps> 0", file=f)
-            i = 1
-            for symb in vocab.symbols[vocab.nspecial :]:
-                if not symb.startswith("madeupword"):
-                    print(f"{symb} {i}", file=f)
-                    i += 1
-    return in_units_file
-
-
-def create_lexicon(
-    cfg: KaldiInitializerConfig,
-    fst_dir: Path,
-    unique_label: str,
-    in_units_file: Path,
-    out_words_file: Path,
-) -> (Path, Path):
-
-    disambig_in_units_file = fst_dir / f"kaldi_dict.{cfg.in_labels}_disambig.txt"
-    lexicon_file = fst_dir / f"kaldi_lexicon.{unique_label}.txt"
-    disambig_lexicon_file = fst_dir / f"kaldi_lexicon.{unique_label}_disambig.txt"
-    if (
-        not lexicon_file.exists()
-        or not disambig_lexicon_file.exists()
-        or not disambig_in_units_file.exists()
-    ):
-        logger.info(f"Creating {lexicon_file} (in units file: {in_units_file})")
-
-        assert cfg.wav2letter_lexicon is not None or cfg.in_labels == cfg.out_labels
-
-        if cfg.wav2letter_lexicon is not None:
-            lm_words = set()
-            with open(out_words_file, "r") as lm_dict_f:
-                for line in lm_dict_f:
-                    lm_words.add(line.split()[0])
-
-            num_skipped = 0
-            total = 0
-            with open(cfg.wav2letter_lexicon, "r") as w2l_lex_f, open(
-                lexicon_file, "w"
-            ) as out_f:
-                for line in w2l_lex_f:
-                    items = line.rstrip().split("\t")
-                    assert len(items) == 2, items
-                    if items[0] in lm_words:
-                        print(items[0], items[1], file=out_f)
-                    else:
-                        num_skipped += 1
-                        logger.debug(
-                            f"Skipping word {items[0]} as it was not found in LM"
-                        )
-                    total += 1
-            if num_skipped > 0:
-                logger.warning(
-                    f"Skipped {num_skipped} out of {total} words as they were not found in LM"
-                )
-        else:
-            with open(in_units_file, "r") as in_f, open(lexicon_file, "w") as out_f:
-                for line in in_f:
-                    symb = line.split()[0]
-                    if symb != "<eps>" and symb != "<ctc_blank>" and symb != "<SIL>":
-                        print(symb, symb, file=out_f)
-
-        lex_disambig_path = (
-            Path(cfg.kaldi_root) / "egs/wsj/s5/utils/add_lex_disambig.pl"
-        )
-        res = subprocess.run(
-            [lex_disambig_path, lexicon_file, disambig_lexicon_file],
-            check=True,
-            capture_output=True,
-        )
-        ndisambig = int(res.stdout)
-        disamib_path = Path(cfg.kaldi_root) / "egs/wsj/s5/utils/add_disambig.pl"
-        res = subprocess.run(
-            [disamib_path, "--include-zero", in_units_file, str(ndisambig)],
-            check=True,
-            capture_output=True,
-        )
-        with open(disambig_in_units_file, "wb") as f:
-            f.write(res.stdout)
-
-    return disambig_lexicon_file, disambig_in_units_file
-
-
-def create_G(
-    kaldi_root: Path, fst_dir: Path, lm_arpa: Path, arpa_base: str
-) -> (Path, Path):
-
-    out_words_file = fst_dir / f"kaldi_dict.{arpa_base}.txt"
-    grammar_graph = fst_dir / f"G_{arpa_base}.fst"
-    if not grammar_graph.exists() or not out_words_file.exists():
-        logger.info(f"Creating {grammar_graph}")
-        arpa2fst = kaldi_root / "src/lmbin/arpa2fst"
-        subprocess.run(
-            [
-                arpa2fst,
-                "--disambig-symbol=#0",
-                f"--write-symbol-table={out_words_file}",
-                lm_arpa,
-                grammar_graph,
-            ],
-            check=True,
-        )
-    return grammar_graph, out_words_file
-
-
-def create_L(
-    kaldi_root: Path,
-    fst_dir: Path,
-    unique_label: str,
-    lexicon_file: Path,
-    in_units_file: Path,
-    out_words_file: Path,
-) -> Path:
-    lexicon_graph = fst_dir / f"L.{unique_label}.fst"
-
-    if not lexicon_graph.exists():
-        logger.info(f"Creating {lexicon_graph} (in units: {in_units_file})")
-        make_lex = kaldi_root / "egs/wsj/s5/utils/make_lexicon_fst.pl"
-        fstcompile = kaldi_root / "tools/openfst-1.6.7/bin/fstcompile"
-        fstaddselfloops = kaldi_root / "src/fstbin/fstaddselfloops"
-        fstarcsort = kaldi_root / "tools/openfst-1.6.7/bin/fstarcsort"
-
-        def write_disambig_symbol(file):
-            with open(file, "r") as f:
-                for line in f:
-                    items = line.rstrip().split()
-                    if items[0] == "#0":
-                        out_path = str(file) + "_disamig"
-                        with open(out_path, "w") as out_f:
-                            print(items[1], file=out_f)
-                            return out_path
-
-            return None
-
-        in_disambig_sym = write_disambig_symbol(in_units_file)
-        assert in_disambig_sym is not None
-        out_disambig_sym = write_disambig_symbol(out_words_file)
-        assert out_disambig_sym is not None
-
-        try:
-            with open(lexicon_graph, "wb") as out_f:
-                res = subprocess.run(
-                    [make_lex, lexicon_file], capture_output=True, check=True
-                )
-                assert len(res.stderr) == 0, res.stderr.decode("utf-8")
-                res = subprocess.run(
-                    [
-                        fstcompile,
-                        f"--isymbols={in_units_file}",
-                        f"--osymbols={out_words_file}",
-                        "--keep_isymbols=false",
-                        "--keep_osymbols=false",
-                    ],
-                    input=res.stdout,
-                    capture_output=True,
-                )
-                assert len(res.stderr) == 0, res.stderr.decode("utf-8")
-                res = subprocess.run(
-                    [fstaddselfloops, in_disambig_sym, out_disambig_sym],
-                    input=res.stdout,
-                    capture_output=True,
-                    check=True,
-                )
-                res = subprocess.run(
-                    [fstarcsort, "--sort_type=olabel"],
-                    input=res.stdout,
-                    capture_output=True,
-                    check=True,
-                )
-                out_f.write(res.stdout)
-        except subprocess.CalledProcessError as e:
-            logger.error(f"cmd: {e.cmd}, err: {e.stderr.decode('utf-8')}")
-            os.remove(lexicon_graph)
-            raise
-        except AssertionError:
-            os.remove(lexicon_graph)
-            raise
-
-    return lexicon_graph
-
-
-def create_LG(
-    kaldi_root: Path,
-    fst_dir: Path,
-    unique_label: str,
-    lexicon_graph: Path,
-    grammar_graph: Path,
-) -> Path:
-    lg_graph = fst_dir / f"LG.{unique_label}.fst"
-
-    if not lg_graph.exists():
-        logger.info(f"Creating {lg_graph}")
-
-        fsttablecompose = kaldi_root / "src/fstbin/fsttablecompose"
-        fstdeterminizestar = kaldi_root / "src/fstbin/fstdeterminizestar"
-        fstminimizeencoded = kaldi_root / "src/fstbin/fstminimizeencoded"
-        fstpushspecial = kaldi_root / "src/fstbin/fstpushspecial"
-        fstarcsort = kaldi_root / "tools/openfst-1.6.7/bin/fstarcsort"
-
-        try:
-            with open(lg_graph, "wb") as out_f:
-                res = subprocess.run(
-                    [fsttablecompose, lexicon_graph, grammar_graph],
-                    capture_output=True,
-                    check=True,
-                )
-                res = subprocess.run(
-                    [
-                        fstdeterminizestar,
-                        "--use-log=true",
-                    ],
-                    input=res.stdout,
-                    capture_output=True,
-                )
-                res = subprocess.run(
-                    [fstminimizeencoded],
-                    input=res.stdout,
-                    capture_output=True,
-                    check=True,
-                )
-                res = subprocess.run(
-                    [fstpushspecial],
-                    input=res.stdout,
-                    capture_output=True,
-                    check=True,
-                )
-                res = subprocess.run(
-                    [fstarcsort, "--sort_type=ilabel"],
-                    input=res.stdout,
-                    capture_output=True,
-                    check=True,
-                )
-                out_f.write(res.stdout)
-        except subprocess.CalledProcessError as e:
-            logger.error(f"cmd: {e.cmd}, err: {e.stderr.decode('utf-8')}")
-            os.remove(lg_graph)
-            raise
-
-    return lg_graph
-
-
-def create_H(
-    kaldi_root: Path,
-    fst_dir: Path,
-    disambig_out_units_file: Path,
-    in_labels: str,
-    vocab: Dictionary,
-    blk_sym: str,
-    silence_symbol: Optional[str],
-) -> (Path, Path, Path):
-    h_graph = (
-        fst_dir / f"H.{in_labels}{'_' + silence_symbol if silence_symbol else ''}.fst"
-    )
-    h_out_units_file = fst_dir / f"kaldi_dict.h_out.{in_labels}.txt"
-    disambig_in_units_file_int = Path(str(h_graph) + "isym_disambig.int")
-    disambig_out_units_file_int = Path(str(disambig_out_units_file) + ".int")
-    if (
-        not h_graph.exists()
-        or not h_out_units_file.exists()
-        or not disambig_in_units_file_int.exists()
-    ):
-        logger.info(f"Creating {h_graph}")
-        eps_sym = "<eps>"
-
-        num_disambig = 0
-        osymbols = []
-
-        with open(disambig_out_units_file, "r") as f, open(
-            disambig_out_units_file_int, "w"
-        ) as out_f:
-            for line in f:
-                symb, id = line.rstrip().split()
-                if line.startswith("#"):
-                    num_disambig += 1
-                    print(id, file=out_f)
-                else:
-                    if len(osymbols) == 0:
-                        assert symb == eps_sym, symb
-                    osymbols.append((symb, id))
-
-        i_idx = 0
-        isymbols = [(eps_sym, 0)]
-
-        imap = {}
-
-        for i, s in enumerate(vocab.symbols):
-            i_idx += 1
-            isymbols.append((s, i_idx))
-            imap[s] = i_idx
-
-        fst_str = []
-
-        node_idx = 0
-        root_node = node_idx
-
-        special_symbols = [blk_sym]
-        if silence_symbol is not None:
-            special_symbols.append(silence_symbol)
-
-        for ss in special_symbols:
-            fst_str.append("{} {} {} {}".format(root_node, root_node, ss, eps_sym))
-
-        for symbol, _ in osymbols:
-            if symbol == eps_sym or symbol.startswith("#"):
-                continue
-
-            node_idx += 1
-            # 1. from root to emitting state
-            fst_str.append("{} {} {} {}".format(root_node, node_idx, symbol, symbol))
-            # 2. from emitting state back to root
-            fst_str.append("{} {} {} {}".format(node_idx, root_node, eps_sym, eps_sym))
-            # 3. from emitting state to optional blank state
-            pre_node = node_idx
-            node_idx += 1
-            for ss in special_symbols:
-                fst_str.append("{} {} {} {}".format(pre_node, node_idx, ss, eps_sym))
-            # 4. from blank state back to root
-            fst_str.append("{} {} {} {}".format(node_idx, root_node, eps_sym, eps_sym))
-
-        fst_str.append("{}".format(root_node))
-
-        fst_str = "\n".join(fst_str)
-        h_str = str(h_graph)
-        isym_file = h_str + ".isym"
-
-        with open(isym_file, "w") as f:
-            for sym, id in isymbols:
-                f.write("{} {}\n".format(sym, id))
-
-        with open(h_out_units_file, "w") as f:
-            for sym, id in osymbols:
-                f.write("{} {}\n".format(sym, id))
-
-        with open(disambig_in_units_file_int, "w") as f:
-            disam_sym_id = len(isymbols)
-            for _ in range(num_disambig):
-                f.write("{}\n".format(disam_sym_id))
-                disam_sym_id += 1
-
-        fstcompile = kaldi_root / "tools/openfst-1.6.7/bin/fstcompile"
-        fstaddselfloops = kaldi_root / "src/fstbin/fstaddselfloops"
-        fstarcsort = kaldi_root / "tools/openfst-1.6.7/bin/fstarcsort"
-
-        try:
-            with open(h_graph, "wb") as out_f:
-                res = subprocess.run(
-                    [
-                        fstcompile,
-                        f"--isymbols={isym_file}",
-                        f"--osymbols={h_out_units_file}",
-                        "--keep_isymbols=false",
-                        "--keep_osymbols=false",
-                    ],
-                    input=str.encode(fst_str),
-                    capture_output=True,
-                    check=True,
-                )
-                res = subprocess.run(
-                    [
-                        fstaddselfloops,
-                        disambig_in_units_file_int,
-                        disambig_out_units_file_int,
-                    ],
-                    input=res.stdout,
-                    capture_output=True,
-                    check=True,
-                )
-                res = subprocess.run(
-                    [fstarcsort, "--sort_type=olabel"],
-                    input=res.stdout,
-                    capture_output=True,
-                    check=True,
-                )
-                out_f.write(res.stdout)
-        except subprocess.CalledProcessError as e:
-            logger.error(f"cmd: {e.cmd}, err: {e.stderr.decode('utf-8')}")
-            os.remove(h_graph)
-            raise
-    return h_graph, h_out_units_file, disambig_in_units_file_int
-
-
-def create_HLGa(
-    kaldi_root: Path,
-    fst_dir: Path,
-    unique_label: str,
-    h_graph: Path,
-    lg_graph: Path,
-    disambig_in_words_file_int: Path,
-) -> Path:
-    hlga_graph = fst_dir / f"HLGa.{unique_label}.fst"
-
-    if not hlga_graph.exists():
-        logger.info(f"Creating {hlga_graph}")
-
-        fsttablecompose = kaldi_root / "src/fstbin/fsttablecompose"
-        fstdeterminizestar = kaldi_root / "src/fstbin/fstdeterminizestar"
-        fstrmsymbols = kaldi_root / "src/fstbin/fstrmsymbols"
-        fstrmepslocal = kaldi_root / "src/fstbin/fstrmepslocal"
-        fstminimizeencoded = kaldi_root / "src/fstbin/fstminimizeencoded"
-
-        try:
-            with open(hlga_graph, "wb") as out_f:
-                res = subprocess.run(
-                    [
-                        fsttablecompose,
-                        h_graph,
-                        lg_graph,
-                    ],
-                    capture_output=True,
-                    check=True,
-                )
-                res = subprocess.run(
-                    [fstdeterminizestar, "--use-log=true"],
-                    input=res.stdout,
-                    capture_output=True,
-                    check=True,
-                )
-                res = subprocess.run(
-                    [fstrmsymbols, disambig_in_words_file_int],
-                    input=res.stdout,
-                    capture_output=True,
-                    check=True,
-                )
-                res = subprocess.run(
-                    [fstrmepslocal],
-                    input=res.stdout,
-                    capture_output=True,
-                    check=True,
-                )
-                res = subprocess.run(
-                    [fstminimizeencoded],
-                    input=res.stdout,
-                    capture_output=True,
-                    check=True,
-                )
-                out_f.write(res.stdout)
-        except subprocess.CalledProcessError as e:
-            logger.error(f"cmd: {e.cmd}, err: {e.stderr.decode('utf-8')}")
-            os.remove(hlga_graph)
-            raise
-
-    return hlga_graph
-
-
-def create_HLa(
-    kaldi_root: Path,
-    fst_dir: Path,
-    unique_label: str,
-    h_graph: Path,
-    l_graph: Path,
-    disambig_in_words_file_int: Path,
-) -> Path:
-    hla_graph = fst_dir / f"HLa.{unique_label}.fst"
-
-    if not hla_graph.exists():
-        logger.info(f"Creating {hla_graph}")
-
-        fsttablecompose = kaldi_root / "src/fstbin/fsttablecompose"
-        fstdeterminizestar = kaldi_root / "src/fstbin/fstdeterminizestar"
-        fstrmsymbols = kaldi_root / "src/fstbin/fstrmsymbols"
-        fstrmepslocal = kaldi_root / "src/fstbin/fstrmepslocal"
-        fstminimizeencoded = kaldi_root / "src/fstbin/fstminimizeencoded"
-
-        try:
-            with open(hla_graph, "wb") as out_f:
-                res = subprocess.run(
-                    [
-                        fsttablecompose,
-                        h_graph,
-                        l_graph,
-                    ],
-                    capture_output=True,
-                    check=True,
-                )
-                res = subprocess.run(
-                    [fstdeterminizestar, "--use-log=true"],
-                    input=res.stdout,
-                    capture_output=True,
-                    check=True,
-                )
-                res = subprocess.run(
-                    [fstrmsymbols, disambig_in_words_file_int],
-                    input=res.stdout,
-                    capture_output=True,
-                    check=True,
-                )
-                res = subprocess.run(
-                    [fstrmepslocal],
-                    input=res.stdout,
-                    capture_output=True,
-                    check=True,
-                )
-                res = subprocess.run(
-                    [fstminimizeencoded],
-                    input=res.stdout,
-                    capture_output=True,
-                    check=True,
-                )
-                out_f.write(res.stdout)
-        except subprocess.CalledProcessError as e:
-            logger.error(f"cmd: {e.cmd}, err: {e.stderr.decode('utf-8')}")
-            os.remove(hla_graph)
-            raise
-
-    return hla_graph
-
-
-def create_HLG(
-    kaldi_root: Path,
-    fst_dir: Path,
-    unique_label: str,
-    hlga_graph: Path,
-    prefix: str = "HLG",
-) -> Path:
-    hlg_graph = fst_dir / f"{prefix}.{unique_label}.fst"
-
-    if not hlg_graph.exists():
-        logger.info(f"Creating {hlg_graph}")
-
-        add_self_loop = script_dir / "add-self-loop-simple"
-        kaldi_src = kaldi_root / "src"
-        kaldi_lib = kaldi_src / "lib"
-
-        try:
-            if not add_self_loop.exists():
-                fst_include = kaldi_root / "tools/openfst-1.6.7/include"
-                add_self_loop_src = script_dir / "add-self-loop-simple.cc"
-
-                subprocess.run(
-                    [
-                        "c++",
-                        f"-I{kaldi_src}",
-                        f"-I{fst_include}",
-                        f"-L{kaldi_lib}",
-                        add_self_loop_src,
-                        "-lkaldi-base",
-                        "-lkaldi-fstext",
-                        "-o",
-                        add_self_loop,
-                    ],
-                    check=True,
-                )
-
-            my_env = os.environ.copy()
-            my_env["LD_LIBRARY_PATH"] = f"{kaldi_lib}:{my_env['LD_LIBRARY_PATH']}"
-
-            subprocess.run(
-                [
-                    add_self_loop,
-                    hlga_graph,
-                    hlg_graph,
-                ],
-                check=True,
-                capture_output=True,
-                env=my_env,
-            )
-        except subprocess.CalledProcessError as e:
-            logger.error(f"cmd: {e.cmd}, err: {e.stderr.decode('utf-8')}")
-            raise
-
-    return hlg_graph
-
-
-def initalize_kaldi(cfg: KaldiInitializerConfig) -> Path:
-    if cfg.fst_dir is None:
-        cfg.fst_dir = osp.join(cfg.data_dir, "kaldi")
-    if cfg.out_labels is None:
-        cfg.out_labels = cfg.in_labels
-
-    kaldi_root = Path(cfg.kaldi_root)
-    data_dir = Path(cfg.data_dir)
-    fst_dir = Path(cfg.fst_dir)
-    fst_dir.mkdir(parents=True, exist_ok=True)
-
-    arpa_base = osp.splitext(osp.basename(cfg.lm_arpa))[0]
-    unique_label = f"{cfg.in_labels}.{arpa_base}"
-
-    with open(data_dir / f"dict.{cfg.in_labels}.txt", "r") as f:
-        vocab = Dictionary.load(f)
-
-    in_units_file = create_units(fst_dir, cfg.in_labels, vocab)
-
-    grammar_graph, out_words_file = create_G(
-        kaldi_root, fst_dir, Path(cfg.lm_arpa), arpa_base
-    )
-
-    disambig_lexicon_file, disambig_L_in_units_file = create_lexicon(
-        cfg, fst_dir, unique_label, in_units_file, out_words_file
-    )
-
-    h_graph, h_out_units_file, disambig_in_units_file_int = create_H(
-        kaldi_root,
-        fst_dir,
-        disambig_L_in_units_file,
-        cfg.in_labels,
-        vocab,
-        cfg.blank_symbol,
-        cfg.silence_symbol,
-    )
-    lexicon_graph = create_L(
-        kaldi_root,
-        fst_dir,
-        unique_label,
-        disambig_lexicon_file,
-        disambig_L_in_units_file,
-        out_words_file,
-    )
-    lg_graph = create_LG(
-        kaldi_root, fst_dir, unique_label, lexicon_graph, grammar_graph
-    )
-    hlga_graph = create_HLGa(
-        kaldi_root, fst_dir, unique_label, h_graph, lg_graph, disambig_in_units_file_int
-    )
-    hlg_graph = create_HLG(kaldi_root, fst_dir, unique_label, hlga_graph)
-
-    # for debugging
-    # hla_graph = create_HLa(kaldi_root, fst_dir, unique_label, h_graph, lexicon_graph, disambig_in_units_file_int)
-    # hl_graph = create_HLG(kaldi_root, fst_dir, unique_label, hla_graph, prefix="HL_looped")
-    # create_HLG(kaldi_root, fst_dir, "phnc", h_graph, prefix="H_looped")
-
-    return hlg_graph
-
-
-@hydra.main(config_path=config_path, config_name="kaldi_initializer")
-def cli_main(cfg: KaldiInitializerConfig) -> None:
-    container = OmegaConf.to_container(cfg, resolve=True, enum_to_str=True)
-    cfg = OmegaConf.create(container)
-    OmegaConf.set_struct(cfg, True)
-    initalize_kaldi(cfg)
-
-
-if __name__ == "__main__":
-
-    logging.root.setLevel(logging.INFO)
-    logging.basicConfig(level=logging.INFO)
-
-    try:
-        from hydra._internal.utils import (  # pylint: disable=import-outside-toplevel
-            get_args,
-        )
-
-        cfg_name = get_args().config_name or "kaldi_initializer"
-    except ImportError:
-        logger.warning("Failed to get config name from hydra args")
-        cfg_name = "kaldi_initializer"
-
-    cs = ConfigStore.instance()
-    cs.store(name=cfg_name, node=KaldiInitializerConfig)
-
-    cli_main()

audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/models/__init__.py

@@ -1,7 +0,0 @@
-import importlib
-import os
-
-for file in sorted(os.listdir(os.path.dirname(__file__))):
-    if file.endswith(".py") and not file.startswith("_"):
-        model_name = file[: file.find(".py")]
-        importlib.import_module("examples.speech_recognition.models." + model_name)

audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/models/vggtransformer.py

@@ -1,1020 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-import argparse
-import math
-from collections.abc import Iterable
-
-import torch
-import torch.nn as nn
-from examples.speech_recognition.data.data_utils import lengths_to_encoder_padding_mask
-from fairseq import utils
-from fairseq.models import (
-    FairseqEncoder,
-    FairseqEncoderDecoderModel,
-    FairseqEncoderModel,
-    FairseqIncrementalDecoder,
-    register_model,
-    register_model_architecture,
-)
-from fairseq.modules import (
-    LinearizedConvolution,
-    TransformerDecoderLayer,
-    TransformerEncoderLayer,
-    VGGBlock,
-)
-
-
-@register_model("asr_vggtransformer")
-class VGGTransformerModel(FairseqEncoderDecoderModel):
-    """
-    Transformers with convolutional context for ASR
-    https://arxiv.org/abs/1904.11660
-    """
-
-    def __init__(self, encoder, decoder):
-        super().__init__(encoder, decoder)
-
-    @staticmethod
-    def add_args(parser):
-        """Add model-specific arguments to the parser."""
-        parser.add_argument(
-            "--input-feat-per-channel",
-            type=int,
-            metavar="N",
-            help="encoder input dimension per input channel",
-        )
-        parser.add_argument(
-            "--vggblock-enc-config",
-            type=str,
-            metavar="EXPR",
-            help="""
-    an array of tuples each containing the configuration of one vggblock:
-    [(out_channels,
-      conv_kernel_size,
-      pooling_kernel_size,
-      num_conv_layers,
-      use_layer_norm), ...])
-            """,
-        )
-        parser.add_argument(
-            "--transformer-enc-config",
-            type=str,
-            metavar="EXPR",
-            help=""""
-    a tuple containing the configuration of the encoder transformer layers
-    configurations:
-    [(input_dim,
-      num_heads,
-      ffn_dim,
-      normalize_before,
-      dropout,
-      attention_dropout,
-      relu_dropout), ...]')
-            """,
-        )
-        parser.add_argument(
-            "--enc-output-dim",
-            type=int,
-            metavar="N",
-            help="""
-    encoder output dimension, can be None. If specified, projecting the
-    transformer output to the specified dimension""",
-        )
-        parser.add_argument(
-            "--in-channels",
-            type=int,
-            metavar="N",
-            help="number of encoder input channels",
-        )
-        parser.add_argument(
-            "--tgt-embed-dim",
-            type=int,
-            metavar="N",
-            help="embedding dimension of the decoder target tokens",
-        )
-        parser.add_argument(
-            "--transformer-dec-config",
-            type=str,
-            metavar="EXPR",
-            help="""
-    a tuple containing the configuration of the decoder transformer layers
-    configurations:
-    [(input_dim,
-      num_heads,
-      ffn_dim,
-      normalize_before,
-      dropout,
-      attention_dropout,
-      relu_dropout), ...]
-            """,
-        )
-        parser.add_argument(
-            "--conv-dec-config",
-            type=str,
-            metavar="EXPR",
-            help="""
-    an array of tuples for the decoder 1-D convolution config
-        [(out_channels, conv_kernel_size, use_layer_norm), ...]""",
-        )
-
-    @classmethod
-    def build_encoder(cls, args, task):
-        return VGGTransformerEncoder(
-            input_feat_per_channel=args.input_feat_per_channel,
-            vggblock_config=eval(args.vggblock_enc_config),
-            transformer_config=eval(args.transformer_enc_config),
-            encoder_output_dim=args.enc_output_dim,
-            in_channels=args.in_channels,
-        )
-
-    @classmethod
-    def build_decoder(cls, args, task):
-        return TransformerDecoder(
-            dictionary=task.target_dictionary,
-            embed_dim=args.tgt_embed_dim,
-            transformer_config=eval(args.transformer_dec_config),
-            conv_config=eval(args.conv_dec_config),
-            encoder_output_dim=args.enc_output_dim,
-        )
-
-    @classmethod
-    def build_model(cls, args, task):
-        """Build a new model instance."""
-        # make sure that all args are properly defaulted
-        # (in case there are any new ones)
-        base_architecture(args)
-
-        encoder = cls.build_encoder(args, task)
-        decoder = cls.build_decoder(args, task)
-        return cls(encoder, decoder)
-
-    def get_normalized_probs(self, net_output, log_probs, sample=None):
-        # net_output['encoder_out'] is a (B, T, D) tensor
-        lprobs = super().get_normalized_probs(net_output, log_probs, sample)
-        lprobs.batch_first = True
-        return lprobs
-
-
-DEFAULT_ENC_VGGBLOCK_CONFIG = ((32, 3, 2, 2, False),) * 2
-DEFAULT_ENC_TRANSFORMER_CONFIG = ((256, 4, 1024, True, 0.2, 0.2, 0.2),) * 2
-# 256: embedding dimension
-# 4: number of heads
-# 1024: FFN
-# True: apply layerNorm before (dropout + resiaul) instead of after
-# 0.2 (dropout): dropout after MultiheadAttention and second FC
-# 0.2 (attention_dropout): dropout in MultiheadAttention
-# 0.2 (relu_dropout): dropout after ReLu
-DEFAULT_DEC_TRANSFORMER_CONFIG = ((256, 2, 1024, True, 0.2, 0.2, 0.2),) * 2
-DEFAULT_DEC_CONV_CONFIG = ((256, 3, True),) * 2
-
-
-# TODO: repace transformer encoder config from one liner
-# to explicit args to get rid of this transformation
-def prepare_transformer_encoder_params(
-    input_dim,
-    num_heads,
-    ffn_dim,
-    normalize_before,
-    dropout,
-    attention_dropout,
-    relu_dropout,
-):
-    args = argparse.Namespace()
-    args.encoder_embed_dim = input_dim
-    args.encoder_attention_heads = num_heads
-    args.attention_dropout = attention_dropout
-    args.dropout = dropout
-    args.activation_dropout = relu_dropout
-    args.encoder_normalize_before = normalize_before
-    args.encoder_ffn_embed_dim = ffn_dim
-    return args
-
-
-def prepare_transformer_decoder_params(
-    input_dim,
-    num_heads,
-    ffn_dim,
-    normalize_before,
-    dropout,
-    attention_dropout,
-    relu_dropout,
-):
-    args = argparse.Namespace()
-    args.encoder_embed_dim = None
-    args.decoder_embed_dim = input_dim
-    args.decoder_attention_heads = num_heads
-    args.attention_dropout = attention_dropout
-    args.dropout = dropout
-    args.activation_dropout = relu_dropout
-    args.decoder_normalize_before = normalize_before
-    args.decoder_ffn_embed_dim = ffn_dim
-    return args
-
-
-class VGGTransformerEncoder(FairseqEncoder):
-    """VGG + Transformer encoder"""
-
-    def __init__(
-        self,
-        input_feat_per_channel,
-        vggblock_config=DEFAULT_ENC_VGGBLOCK_CONFIG,
-        transformer_config=DEFAULT_ENC_TRANSFORMER_CONFIG,
-        encoder_output_dim=512,
-        in_channels=1,
-        transformer_context=None,
-        transformer_sampling=None,
-    ):
-        """constructor for VGGTransformerEncoder
-
-        Args:
-            - input_feat_per_channel: feature dim (not including stacked,
-              just base feature)
-            - in_channel: # input channels (e.g., if stack 8 feature vector
-                together, this is 8)
-            - vggblock_config: configuration of vggblock, see comments on
-                DEFAULT_ENC_VGGBLOCK_CONFIG
-            - transformer_config: configuration of transformer layer, see comments
-                on DEFAULT_ENC_TRANSFORMER_CONFIG
-            - encoder_output_dim: final transformer output embedding dimension
-            - transformer_context: (left, right) if set, self-attention will be focused
-              on (t-left, t+right)
-            - transformer_sampling: an iterable of int, must match with
-              len(transformer_config), transformer_sampling[i] indicates sampling
-              factor for i-th transformer layer, after multihead att and feedfoward
-              part
-        """
-        super().__init__(None)
-
-        self.num_vggblocks = 0
-        if vggblock_config is not None:
-            if not isinstance(vggblock_config, Iterable):
-                raise ValueError("vggblock_config is not iterable")
-            self.num_vggblocks = len(vggblock_config)
-
-        self.conv_layers = nn.ModuleList()
-        self.in_channels = in_channels
-        self.input_dim = input_feat_per_channel
-        self.pooling_kernel_sizes = []
-
-        if vggblock_config is not None:
-            for _, config in enumerate(vggblock_config):
-                (
-                    out_channels,
-                    conv_kernel_size,
-                    pooling_kernel_size,
-                    num_conv_layers,
-                    layer_norm,
-                ) = config
-                self.conv_layers.append(
-                    VGGBlock(
-                        in_channels,
-                        out_channels,
-                        conv_kernel_size,
-                        pooling_kernel_size,
-                        num_conv_layers,
-                        input_dim=input_feat_per_channel,
-                        layer_norm=layer_norm,
-                    )
-                )
-                self.pooling_kernel_sizes.append(pooling_kernel_size)
-                in_channels = out_channels
-                input_feat_per_channel = self.conv_layers[-1].output_dim
-
-        transformer_input_dim = self.infer_conv_output_dim(
-            self.in_channels, self.input_dim
-        )
-        # transformer_input_dim is the output dimension of VGG part
-
-        self.validate_transformer_config(transformer_config)
-        self.transformer_context = self.parse_transformer_context(transformer_context)
-        self.transformer_sampling = self.parse_transformer_sampling(
-            transformer_sampling, len(transformer_config)
-        )
-
-        self.transformer_layers = nn.ModuleList()
-
-        if transformer_input_dim != transformer_config[0][0]:
-            self.transformer_layers.append(
-                Linear(transformer_input_dim, transformer_config[0][0])
-            )
-        self.transformer_layers.append(
-            TransformerEncoderLayer(
-                prepare_transformer_encoder_params(*transformer_config[0])
-            )
-        )
-
-        for i in range(1, len(transformer_config)):
-            if transformer_config[i - 1][0] != transformer_config[i][0]:
-                self.transformer_layers.append(
-                    Linear(transformer_config[i - 1][0], transformer_config[i][0])
-                )
-            self.transformer_layers.append(
-                TransformerEncoderLayer(
-                    prepare_transformer_encoder_params(*transformer_config[i])
-                )
-            )
-
-        self.encoder_output_dim = encoder_output_dim
-        self.transformer_layers.extend(
-            [
-                Linear(transformer_config[-1][0], encoder_output_dim),
-                LayerNorm(encoder_output_dim),
-            ]
-        )
-
-    def forward(self, src_tokens, src_lengths, **kwargs):
-        """
-        src_tokens: padded tensor (B, T, C * feat)
-        src_lengths: tensor of original lengths of input utterances (B,)
-        """
-        bsz, max_seq_len, _ = src_tokens.size()
-        x = src_tokens.view(bsz, max_seq_len, self.in_channels, self.input_dim)
-        x = x.transpose(1, 2).contiguous()
-        # (B, C, T, feat)
-
-        for layer_idx in range(len(self.conv_layers)):
-            x = self.conv_layers[layer_idx](x)
-
-        bsz, _, output_seq_len, _ = x.size()
-
-        # (B, C, T, feat) -> (B, T, C, feat) -> (T, B, C, feat) -> (T, B, C * feat)
-        x = x.transpose(1, 2).transpose(0, 1)
-        x = x.contiguous().view(output_seq_len, bsz, -1)
-
-        input_lengths = src_lengths.clone()
-        for s in self.pooling_kernel_sizes:
-            input_lengths = (input_lengths.float() / s).ceil().long()
-
-        encoder_padding_mask, _ = lengths_to_encoder_padding_mask(
-            input_lengths, batch_first=True
-        )
-        if not encoder_padding_mask.any():
-            encoder_padding_mask = None
-
-        subsampling_factor = int(max_seq_len * 1.0 / output_seq_len + 0.5)
-        attn_mask = self.lengths_to_attn_mask(input_lengths, subsampling_factor)
-
-        transformer_layer_idx = 0
-
-        for layer_idx in range(len(self.transformer_layers)):
-
-            if isinstance(self.transformer_layers[layer_idx], TransformerEncoderLayer):
-                x = self.transformer_layers[layer_idx](
-                    x, encoder_padding_mask, attn_mask
-                )
-
-                if self.transformer_sampling[transformer_layer_idx] != 1:
-                    sampling_factor = self.transformer_sampling[transformer_layer_idx]
-                    x, encoder_padding_mask, attn_mask = self.slice(
-                        x, encoder_padding_mask, attn_mask, sampling_factor
-                    )
-
-                transformer_layer_idx += 1
-
-            else:
-                x = self.transformer_layers[layer_idx](x)
-
-        # encoder_padding_maks is a (T x B) tensor, its [t, b] elements indicate
-        # whether encoder_output[t, b] is valid or not (valid=0, invalid=1)
-
-        return {
-            "encoder_out": x,  # (T, B, C)
-            "encoder_padding_mask": encoder_padding_mask.t()
-            if encoder_padding_mask is not None
-            else None,
-            # (B, T) --> (T, B)
-        }
-
-    def infer_conv_output_dim(self, in_channels, input_dim):
-        sample_seq_len = 200
-        sample_bsz = 10
-        x = torch.randn(sample_bsz, in_channels, sample_seq_len, input_dim)
-        for i, _ in enumerate(self.conv_layers):
-            x = self.conv_layers[i](x)
-        x = x.transpose(1, 2)
-        mb, seq = x.size()[:2]
-        return x.contiguous().view(mb, seq, -1).size(-1)
-
-    def validate_transformer_config(self, transformer_config):
-        for config in transformer_config:
-            input_dim, num_heads = config[:2]
-            if input_dim % num_heads != 0:
-                msg = (
-                    "ERROR in transformer config {}: ".format(config)
-                    + "input dimension {} ".format(input_dim)
-                    + "not dividable by number of heads {}".format(num_heads)
-                )
-                raise ValueError(msg)
-
-    def parse_transformer_context(self, transformer_context):
-        """
-        transformer_context can be the following:
-        -   None; indicates no context is used, i.e.,
-            transformer can access full context
-        -   a tuple/list of two int; indicates left and right context,
-            any number <0 indicates infinite context
-                * e.g., (5, 6) indicates that for query at x_t, transformer can
-                access [t-5, t+6] (inclusive)
-                * e.g., (-1, 6) indicates that for query at x_t, transformer can
-                access [0, t+6] (inclusive)
-        """
-        if transformer_context is None:
-            return None
-
-        if not isinstance(transformer_context, Iterable):
-            raise ValueError("transformer context must be Iterable if it is not None")
-
-        if len(transformer_context) != 2:
-            raise ValueError("transformer context must have length 2")
-
-        left_context = transformer_context[0]
-        if left_context < 0:
-            left_context = None
-
-        right_context = transformer_context[1]
-        if right_context < 0:
-            right_context = None
-
-        if left_context is None and right_context is None:
-            return None
-
-        return (left_context, right_context)
-
-    def parse_transformer_sampling(self, transformer_sampling, num_layers):
-        """
-        parsing transformer sampling configuration
-
-        Args:
-            - transformer_sampling, accepted input:
-                * None, indicating no sampling
-                * an Iterable with int (>0) as element
-            - num_layers, expected number of transformer layers, must match with
-              the length of transformer_sampling if it is not None
-
-        Returns:
-            - A tuple with length num_layers
-        """
-        if transformer_sampling is None:
-            return (1,) * num_layers
-
-        if not isinstance(transformer_sampling, Iterable):
-            raise ValueError(
-                "transformer_sampling must be an iterable if it is not None"
-            )
-
-        if len(transformer_sampling) != num_layers:
-            raise ValueError(
-                "transformer_sampling {} does not match with the number "
-                "of layers {}".format(transformer_sampling, num_layers)
-            )
-
-        for layer, value in enumerate(transformer_sampling):
-            if not isinstance(value, int):
-                raise ValueError("Invalid value in transformer_sampling: ")
-            if value < 1:
-                raise ValueError(
-                    "{} layer's subsampling is {}.".format(layer, value)
-                    + " This is not allowed! "
-                )
-        return transformer_sampling
-
-    def slice(self, embedding, padding_mask, attn_mask, sampling_factor):
-        """
-        embedding is a (T, B, D) tensor
-        padding_mask is a (B, T) tensor or None
-        attn_mask is a (T, T) tensor or None
-        """
-        embedding = embedding[::sampling_factor, :, :]
-        if padding_mask is not None:
-            padding_mask = padding_mask[:, ::sampling_factor]
-        if attn_mask is not None:
-            attn_mask = attn_mask[::sampling_factor, ::sampling_factor]
-
-        return embedding, padding_mask, attn_mask
-
-    def lengths_to_attn_mask(self, input_lengths, subsampling_factor=1):
-        """
-        create attention mask according to sequence lengths and transformer
-        context
-
-        Args:
-            - input_lengths: (B, )-shape Int/Long tensor; input_lengths[b] is
-              the length of b-th sequence
-            - subsampling_factor: int
-                * Note that the left_context and right_context is specified in
-                  the input frame-level while input to transformer may already
-                  go through subsampling (e.g., the use of striding in vggblock)
-                  we use subsampling_factor to scale the left/right context
-
-        Return:
-            - a (T, T) binary tensor or None, where T is max(input_lengths)
-                * if self.transformer_context is None, None
-                * if left_context is None,
-                    * attn_mask[t, t + right_context + 1:] = 1
-                    * others = 0
-                * if right_context is None,
-                    * attn_mask[t, 0:t - left_context] = 1
-                    * others = 0
-                * elsif
-                    * attn_mask[t, t - left_context: t + right_context + 1] = 0
-                    * others = 1
-        """
-        if self.transformer_context is None:
-            return None
-
-        maxT = torch.max(input_lengths).item()
-        attn_mask = torch.zeros(maxT, maxT)
-
-        left_context = self.transformer_context[0]
-        right_context = self.transformer_context[1]
-        if left_context is not None:
-            left_context = math.ceil(self.transformer_context[0] / subsampling_factor)
-        if right_context is not None:
-            right_context = math.ceil(self.transformer_context[1] / subsampling_factor)
-
-        for t in range(maxT):
-            if left_context is not None:
-                st = 0
-                en = max(st, t - left_context)
-                attn_mask[t, st:en] = 1
-            if right_context is not None:
-                st = t + right_context + 1
-                st = min(st, maxT - 1)
-                attn_mask[t, st:] = 1
-
-        return attn_mask.to(input_lengths.device)
-
-    def reorder_encoder_out(self, encoder_out, new_order):
-        encoder_out["encoder_out"] = encoder_out["encoder_out"].index_select(
-            1, new_order
-        )
-        if encoder_out["encoder_padding_mask"] is not None:
-            encoder_out["encoder_padding_mask"] = encoder_out[
-                "encoder_padding_mask"
-            ].index_select(1, new_order)
-        return encoder_out
-
-
-class TransformerDecoder(FairseqIncrementalDecoder):
-    """
-    Transformer decoder consisting of *args.decoder_layers* layers. Each layer
-    is a :class:`TransformerDecoderLayer`.
-    Args:
-        args (argparse.Namespace): parsed command-line arguments
-        dictionary (~fairseq.data.Dictionary): decoding dictionary
-        embed_tokens (torch.nn.Embedding): output embedding
-        no_encoder_attn (bool, optional): whether to attend to encoder outputs.
-            Default: ``False``
-        left_pad (bool, optional): whether the input is left-padded. Default:
-            ``False``
-    """
-
-    def __init__(
-        self,
-        dictionary,
-        embed_dim=512,
-        transformer_config=DEFAULT_ENC_TRANSFORMER_CONFIG,
-        conv_config=DEFAULT_DEC_CONV_CONFIG,
-        encoder_output_dim=512,
-    ):
-
-        super().__init__(dictionary)
-        vocab_size = len(dictionary)
-        self.padding_idx = dictionary.pad()
-        self.embed_tokens = Embedding(vocab_size, embed_dim, self.padding_idx)
-
-        self.conv_layers = nn.ModuleList()
-        for i in range(len(conv_config)):
-            out_channels, kernel_size, layer_norm = conv_config[i]
-            if i == 0:
-                conv_layer = LinearizedConv1d(
-                    embed_dim, out_channels, kernel_size, padding=kernel_size - 1
-                )
-            else:
-                conv_layer = LinearizedConv1d(
-                    conv_config[i - 1][0],
-                    out_channels,
-                    kernel_size,
-                    padding=kernel_size - 1,
-                )
-            self.conv_layers.append(conv_layer)
-            if layer_norm:
-                self.conv_layers.append(nn.LayerNorm(out_channels))
-            self.conv_layers.append(nn.ReLU())
-
-        self.layers = nn.ModuleList()
-        if conv_config[-1][0] != transformer_config[0][0]:
-            self.layers.append(Linear(conv_config[-1][0], transformer_config[0][0]))
-        self.layers.append(
-            TransformerDecoderLayer(
-                prepare_transformer_decoder_params(*transformer_config[0])
-            )
-        )
-
-        for i in range(1, len(transformer_config)):
-            if transformer_config[i - 1][0] != transformer_config[i][0]:
-                self.layers.append(
-                    Linear(transformer_config[i - 1][0], transformer_config[i][0])
-                )
-            self.layers.append(
-                TransformerDecoderLayer(
-                    prepare_transformer_decoder_params(*transformer_config[i])
-                )
-            )
-        self.fc_out = Linear(transformer_config[-1][0], vocab_size)
-
-    def forward(self, prev_output_tokens, encoder_out=None, incremental_state=None):
-        """
-        Args:
-            prev_output_tokens (LongTensor): previous decoder outputs of shape
-                `(batch, tgt_len)`, for input feeding/teacher forcing
-            encoder_out (Tensor, optional): output from the encoder, used for
-                encoder-side attention
-            incremental_state (dict): dictionary used for storing state during
-                :ref:`Incremental decoding`
-        Returns:
-            tuple:
-                - the last decoder layer's output of shape `(batch, tgt_len,
-                  vocab)`
-                - the last decoder layer's attention weights of shape `(batch,
-                  tgt_len, src_len)`
-        """
-        target_padding_mask = (
-            (prev_output_tokens == self.padding_idx).to(prev_output_tokens.device)
-            if incremental_state is None
-            else None
-        )
-
-        if incremental_state is not None:
-            prev_output_tokens = prev_output_tokens[:, -1:]
-
-        # embed tokens
-        x = self.embed_tokens(prev_output_tokens)
-
-        # B x T x C -> T x B x C
-        x = self._transpose_if_training(x, incremental_state)
-
-        for layer in self.conv_layers:
-            if isinstance(layer, LinearizedConvolution):
-                x = layer(x, incremental_state)
-            else:
-                x = layer(x)
-
-        # B x T x C -> T x B x C
-        x = self._transpose_if_inference(x, incremental_state)
-
-        # decoder layers
-        for layer in self.layers:
-            if isinstance(layer, TransformerDecoderLayer):
-                x, *_ = layer(
-                    x,
-                    (encoder_out["encoder_out"] if encoder_out is not None else None),
-                    (
-                        encoder_out["encoder_padding_mask"].t()
-                        if encoder_out["encoder_padding_mask"] is not None
-                        else None
-                    ),
-                    incremental_state,
-                    self_attn_mask=(
-                        self.buffered_future_mask(x)
-                        if incremental_state is None
-                        else None
-                    ),
-                    self_attn_padding_mask=(
-                        target_padding_mask if incremental_state is None else None
-                    ),
-                )
-            else:
-                x = layer(x)
-
-        # T x B x C -> B x T x C
-        x = x.transpose(0, 1)
-
-        x = self.fc_out(x)
-
-        return x, None
-
-    def buffered_future_mask(self, tensor):
-        dim = tensor.size(0)
-        if (
-            not hasattr(self, "_future_mask")
-            or self._future_mask is None
-            or self._future_mask.device != tensor.device
-        ):
-            self._future_mask = torch.triu(
-                utils.fill_with_neg_inf(tensor.new(dim, dim)), 1
-            )
-        if self._future_mask.size(0) < dim:
-            self._future_mask = torch.triu(
-                utils.fill_with_neg_inf(self._future_mask.resize_(dim, dim)), 1
-            )
-        return self._future_mask[:dim, :dim]
-
-    def _transpose_if_training(self, x, incremental_state):
-        if incremental_state is None:
-            x = x.transpose(0, 1)
-        return x
-
-    def _transpose_if_inference(self, x, incremental_state):
-        if incremental_state:
-            x = x.transpose(0, 1)
-        return x
-
-
-@register_model("asr_vggtransformer_encoder")
-class VGGTransformerEncoderModel(FairseqEncoderModel):
-    def __init__(self, encoder):
-        super().__init__(encoder)
-
-    @staticmethod
-    def add_args(parser):
-        """Add model-specific arguments to the parser."""
-        parser.add_argument(
-            "--input-feat-per-channel",
-            type=int,
-            metavar="N",
-            help="encoder input dimension per input channel",
-        )
-        parser.add_argument(
-            "--vggblock-enc-config",
-            type=str,
-            metavar="EXPR",
-            help="""
-    an array of tuples each containing the configuration of one vggblock
-    [(out_channels, conv_kernel_size, pooling_kernel_size,num_conv_layers), ...]
-    """,
-        )
-        parser.add_argument(
-            "--transformer-enc-config",
-            type=str,
-            metavar="EXPR",
-            help="""
-    a tuple containing the configuration of the Transformer layers
-    configurations:
-    [(input_dim,
-      num_heads,
-      ffn_dim,
-      normalize_before,
-      dropout,
-      attention_dropout,
-      relu_dropout), ]""",
-        )
-        parser.add_argument(
-            "--enc-output-dim",
-            type=int,
-            metavar="N",
-            help="encoder output dimension, projecting the LSTM output",
-        )
-        parser.add_argument(
-            "--in-channels",
-            type=int,
-            metavar="N",
-            help="number of encoder input channels",
-        )
-        parser.add_argument(
-            "--transformer-context",
-            type=str,
-            metavar="EXPR",
-            help="""
-    either None or a tuple of two ints, indicating left/right context a
-    transformer can have access to""",
-        )
-        parser.add_argument(
-            "--transformer-sampling",
-            type=str,
-            metavar="EXPR",
-            help="""
-    either None or a tuple of ints, indicating sampling factor in each layer""",
-        )
-
-    @classmethod
-    def build_model(cls, args, task):
-        """Build a new model instance."""
-        base_architecture_enconly(args)
-        encoder = VGGTransformerEncoderOnly(
-            vocab_size=len(task.target_dictionary),
-            input_feat_per_channel=args.input_feat_per_channel,
-            vggblock_config=eval(args.vggblock_enc_config),
-            transformer_config=eval(args.transformer_enc_config),
-            encoder_output_dim=args.enc_output_dim,
-            in_channels=args.in_channels,
-            transformer_context=eval(args.transformer_context),
-            transformer_sampling=eval(args.transformer_sampling),
-        )
-        return cls(encoder)
-
-    def get_normalized_probs(self, net_output, log_probs, sample=None):
-        # net_output['encoder_out'] is a (T, B, D) tensor
-        lprobs = super().get_normalized_probs(net_output, log_probs, sample)
-        # lprobs is a (T, B, D) tensor
-        # we need to transoose to get (B, T, D) tensor
-        lprobs = lprobs.transpose(0, 1).contiguous()
-        lprobs.batch_first = True
-        return lprobs
-
-
-class VGGTransformerEncoderOnly(VGGTransformerEncoder):
-    def __init__(
-        self,
-        vocab_size,
-        input_feat_per_channel,
-        vggblock_config=DEFAULT_ENC_VGGBLOCK_CONFIG,
-        transformer_config=DEFAULT_ENC_TRANSFORMER_CONFIG,
-        encoder_output_dim=512,
-        in_channels=1,
-        transformer_context=None,
-        transformer_sampling=None,
-    ):
-        super().__init__(
-            input_feat_per_channel=input_feat_per_channel,
-            vggblock_config=vggblock_config,
-            transformer_config=transformer_config,
-            encoder_output_dim=encoder_output_dim,
-            in_channels=in_channels,
-            transformer_context=transformer_context,
-            transformer_sampling=transformer_sampling,
-        )
-        self.fc_out = Linear(self.encoder_output_dim, vocab_size)
-
-    def forward(self, src_tokens, src_lengths, **kwargs):
-        """
-        src_tokens: padded tensor (B, T, C * feat)
-        src_lengths: tensor of original lengths of input utterances (B,)
-        """
-
-        enc_out = super().forward(src_tokens, src_lengths)
-        x = self.fc_out(enc_out["encoder_out"])
-        # x = F.log_softmax(x, dim=-1)
-        # Note: no need this line, because model.get_normalized_prob will call
-        # log_softmax
-        return {
-            "encoder_out": x,  # (T, B, C)
-            "encoder_padding_mask": enc_out["encoder_padding_mask"],  # (T, B)
-        }
-
-    def max_positions(self):
-        """Maximum input length supported by the encoder."""
-        return (1e6, 1e6)  # an arbitrary large number
-
-
-def Embedding(num_embeddings, embedding_dim, padding_idx):
-    m = nn.Embedding(num_embeddings, embedding_dim, padding_idx=padding_idx)
-    # nn.init.uniform_(m.weight, -0.1, 0.1)
-    # nn.init.constant_(m.weight[padding_idx], 0)
-    return m
-
-
-def Linear(in_features, out_features, bias=True, dropout=0):
-    """Linear layer (input: N x T x C)"""
-    m = nn.Linear(in_features, out_features, bias=bias)
-    # m.weight.data.uniform_(-0.1, 0.1)
-    # if bias:
-    #     m.bias.data.uniform_(-0.1, 0.1)
-    return m
-
-
-def LinearizedConv1d(in_channels, out_channels, kernel_size, dropout=0, **kwargs):
-    """Weight-normalized Conv1d layer optimized for decoding"""
-    m = LinearizedConvolution(in_channels, out_channels, kernel_size, **kwargs)
-    std = math.sqrt((4 * (1.0 - dropout)) / (m.kernel_size[0] * in_channels))
-    nn.init.normal_(m.weight, mean=0, std=std)
-    nn.init.constant_(m.bias, 0)
-    return nn.utils.weight_norm(m, dim=2)
-
-
-def LayerNorm(embedding_dim):
-    m = nn.LayerNorm(embedding_dim)
-    return m
-
-
-# seq2seq models
-def base_architecture(args):
-    args.input_feat_per_channel = getattr(args, "input_feat_per_channel", 40)
-    args.vggblock_enc_config = getattr(
-        args, "vggblock_enc_config", DEFAULT_ENC_VGGBLOCK_CONFIG
-    )
-    args.transformer_enc_config = getattr(
-        args, "transformer_enc_config", DEFAULT_ENC_TRANSFORMER_CONFIG
-    )
-    args.enc_output_dim = getattr(args, "enc_output_dim", 512)
-    args.in_channels = getattr(args, "in_channels", 1)
-    args.tgt_embed_dim = getattr(args, "tgt_embed_dim", 128)
-    args.transformer_dec_config = getattr(
-        args, "transformer_dec_config", DEFAULT_ENC_TRANSFORMER_CONFIG
-    )
-    args.conv_dec_config = getattr(args, "conv_dec_config", DEFAULT_DEC_CONV_CONFIG)
-    args.transformer_context = getattr(args, "transformer_context", "None")
-
-
-@register_model_architecture("asr_vggtransformer", "vggtransformer_1")
-def vggtransformer_1(args):
-    args.input_feat_per_channel = getattr(args, "input_feat_per_channel", 80)
-    args.vggblock_enc_config = getattr(
-        args, "vggblock_enc_config", "[(64, 3, 2, 2, True), (128, 3, 2, 2, True)]"
-    )
-    args.transformer_enc_config = getattr(
-        args,
-        "transformer_enc_config",
-        "((1024, 16, 4096, True, 0.15, 0.15, 0.15),) * 14",
-    )
-    args.enc_output_dim = getattr(args, "enc_output_dim", 1024)
-    args.tgt_embed_dim = getattr(args, "tgt_embed_dim", 128)
-    args.conv_dec_config = getattr(args, "conv_dec_config", "((256, 3, True),) * 4")
-    args.transformer_dec_config = getattr(
-        args,
-        "transformer_dec_config",
-        "((1024, 16, 4096, True, 0.15, 0.15, 0.15),) * 4",
-    )
-
-
-@register_model_architecture("asr_vggtransformer", "vggtransformer_2")
-def vggtransformer_2(args):
-    args.input_feat_per_channel = getattr(args, "input_feat_per_channel", 80)
-    args.vggblock_enc_config = getattr(
-        args, "vggblock_enc_config", "[(64, 3, 2, 2, True), (128, 3, 2, 2, True)]"
-    )
-    args.transformer_enc_config = getattr(
-        args,
-        "transformer_enc_config",
-        "((1024, 16, 4096, True, 0.15, 0.15, 0.15),) * 16",
-    )
-    args.enc_output_dim = getattr(args, "enc_output_dim", 1024)
-    args.tgt_embed_dim = getattr(args, "tgt_embed_dim", 512)
-    args.conv_dec_config = getattr(args, "conv_dec_config", "((256, 3, True),) * 4")
-    args.transformer_dec_config = getattr(
-        args,
-        "transformer_dec_config",
-        "((1024, 16, 4096, True, 0.15, 0.15, 0.15),) * 6",
-    )
-
-
-@register_model_architecture("asr_vggtransformer", "vggtransformer_base")
-def vggtransformer_base(args):
-    args.input_feat_per_channel = getattr(args, "input_feat_per_channel", 80)
-    args.vggblock_enc_config = getattr(
-        args, "vggblock_enc_config", "[(64, 3, 2, 2, True), (128, 3, 2, 2, True)]"
-    )
-    args.transformer_enc_config = getattr(
-        args, "transformer_enc_config", "((512, 8, 2048, True, 0.15, 0.15, 0.15),) * 12"
-    )
-
-    args.enc_output_dim = getattr(args, "enc_output_dim", 512)
-    args.tgt_embed_dim = getattr(args, "tgt_embed_dim", 512)
-    args.conv_dec_config = getattr(args, "conv_dec_config", "((256, 3, True),) * 4")
-    args.transformer_dec_config = getattr(
-        args, "transformer_dec_config", "((512, 8, 2048, True, 0.15, 0.15, 0.15),) * 6"
-    )
-    # Size estimations:
-    # Encoder:
-    #   - vggblock param: 64*1*3*3 + 64*64*3*3 + 128*64*3*3  + 128*128*3 = 258K
-    #   Transformer:
-    #   - input dimension adapter: 2560 x 512 -> 1.31M
-    #   - transformer_layers (x12) --> 37.74M
-    #       * MultiheadAttention: 512*512*3 (in_proj) + 512*512 (out_proj) = 1.048M
-    #       * FFN weight: 512*2048*2 = 2.097M
-    #   - output dimension adapter: 512 x 512 -> 0.26 M
-    # Decoder:
-    #   - LinearizedConv1d: 512 * 256 * 3 + 256 * 256 * 3 * 3
-    #   - transformer_layer: (x6) --> 25.16M
-    #        * MultiheadAttention (self-attention): 512*512*3 + 512*512 = 1.048M
-    #        * MultiheadAttention (encoder-attention): 512*512*3 + 512*512 = 1.048M
-    #        * FFN: 512*2048*2 = 2.097M
-    # Final FC:
-    #   - FC: 512*5000 = 256K (assuming vocab size 5K)
-    # In total:
-    #       ~65 M
-
-
-# CTC models
-def base_architecture_enconly(args):
-    args.input_feat_per_channel = getattr(args, "input_feat_per_channel", 40)
-    args.vggblock_enc_config = getattr(
-        args, "vggblock_enc_config", "[(32, 3, 2, 2, True)] * 2"
-    )
-    args.transformer_enc_config = getattr(
-        args, "transformer_enc_config", "((256, 4, 1024, True, 0.2, 0.2, 0.2),) * 2"
-    )
-    args.enc_output_dim = getattr(args, "enc_output_dim", 512)
-    args.in_channels = getattr(args, "in_channels", 1)
-    args.transformer_context = getattr(args, "transformer_context", "None")
-    args.transformer_sampling = getattr(args, "transformer_sampling", "None")
-
-
-@register_model_architecture("asr_vggtransformer_encoder", "vggtransformer_enc_1")
-def vggtransformer_enc_1(args):
-    # vggtransformer_1 is the same as vggtransformer_enc_big, except the number
-    # of layers is increased to 16
-    # keep it here for backward compatiablity purpose
-    args.input_feat_per_channel = getattr(args, "input_feat_per_channel", 80)
-    args.vggblock_enc_config = getattr(
-        args, "vggblock_enc_config", "[(64, 3, 2, 2, True), (128, 3, 2, 2, True)]"
-    )
-    args.transformer_enc_config = getattr(
-        args,
-        "transformer_enc_config",
-        "((1024, 16, 4096, True, 0.15, 0.15, 0.15),) * 16",
-    )
-    args.enc_output_dim = getattr(args, "enc_output_dim", 1024)

audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/models/w2l_conv_glu_enc.py

@@ -1,176 +0,0 @@
-#!/usr/bin/env python3
-
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-import math
-
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-from fairseq.models import (
-    FairseqEncoder,
-    FairseqEncoderModel,
-    register_model,
-    register_model_architecture,
-)
-from fairseq.modules.fairseq_dropout import FairseqDropout
-
-default_conv_enc_config = """[
-    (400, 13, 170, 0.2),
-    (440, 14, 0, 0.214),
-    (484, 15, 0, 0.22898),
-    (532, 16, 0, 0.2450086),
-    (584, 17, 0, 0.262159202),
-    (642, 18, 0, 0.28051034614),
-    (706, 19, 0, 0.30014607037),
-    (776, 20, 0, 0.321156295296),
-    (852, 21, 0, 0.343637235966),
-    (936, 22, 0, 0.367691842484),
-    (1028, 23, 0, 0.393430271458),
-    (1130, 24, 0, 0.42097039046),
-    (1242, 25, 0, 0.450438317792),
-    (1366, 26, 0, 0.481969000038),
-    (1502, 27, 0, 0.51570683004),
-    (1652, 28, 0, 0.551806308143),
-    (1816, 29, 0, 0.590432749713),
-]"""
-
-
-@register_model("asr_w2l_conv_glu_encoder")
-class W2lConvGluEncoderModel(FairseqEncoderModel):
-    def __init__(self, encoder):
-        super().__init__(encoder)
-
-    @staticmethod
-    def add_args(parser):
-        """Add model-specific arguments to the parser."""
-        parser.add_argument(
-            "--input-feat-per-channel",
-            type=int,
-            metavar="N",
-            help="encoder input dimension per input channel",
-        )
-        parser.add_argument(
-            "--in-channels",
-            type=int,
-            metavar="N",
-            help="number of encoder input channels",
-        )
-        parser.add_argument(
-            "--conv-enc-config",
-            type=str,
-            metavar="EXPR",
-            help="""
-    an array of tuples each containing the configuration of one conv layer
-    [(out_channels, kernel_size, padding, dropout), ...]
-            """,
-        )
-
-    @classmethod
-    def build_model(cls, args, task):
-        """Build a new model instance."""
-        conv_enc_config = getattr(args, "conv_enc_config", default_conv_enc_config)
-        encoder = W2lConvGluEncoder(
-            vocab_size=len(task.target_dictionary),
-            input_feat_per_channel=args.input_feat_per_channel,
-            in_channels=args.in_channels,
-            conv_enc_config=eval(conv_enc_config),
-        )
-        return cls(encoder)
-
-    def get_normalized_probs(self, net_output, log_probs, sample=None):
-        lprobs = super().get_normalized_probs(net_output, log_probs, sample)
-        lprobs.batch_first = False
-        return lprobs
-
-
-class W2lConvGluEncoder(FairseqEncoder):
-    def __init__(
-        self, vocab_size, input_feat_per_channel, in_channels, conv_enc_config
-    ):
-        super().__init__(None)
-
-        self.input_dim = input_feat_per_channel
-        if in_channels != 1:
-            raise ValueError("only 1 input channel is currently supported")
-
-        self.conv_layers = nn.ModuleList()
-        self.linear_layers = nn.ModuleList()
-        self.dropouts = []
-        cur_channels = input_feat_per_channel
-
-        for out_channels, kernel_size, padding, dropout in conv_enc_config:
-            layer = nn.Conv1d(cur_channels, out_channels, kernel_size, padding=padding)
-            layer.weight.data.mul_(math.sqrt(3))  # match wav2letter init
-            self.conv_layers.append(nn.utils.weight_norm(layer))
-            self.dropouts.append(
-                FairseqDropout(dropout, module_name=self.__class__.__name__)
-            )
-            if out_channels % 2 != 0:
-                raise ValueError("odd # of out_channels is incompatible with GLU")
-            cur_channels = out_channels // 2  # halved by GLU
-
-        for out_channels in [2 * cur_channels, vocab_size]:
-            layer = nn.Linear(cur_channels, out_channels)
-            layer.weight.data.mul_(math.sqrt(3))
-            self.linear_layers.append(nn.utils.weight_norm(layer))
-            cur_channels = out_channels // 2
-
-    def forward(self, src_tokens, src_lengths, **kwargs):
-
-        """
-        src_tokens: padded tensor (B, T, C * feat)
-        src_lengths: tensor of original lengths of input utterances (B,)
-        """
-        B, T, _ = src_tokens.size()
-        x = src_tokens.transpose(1, 2).contiguous()  # (B, feat, T) assuming C == 1
-
-        for layer_idx in range(len(self.conv_layers)):
-            x = self.conv_layers[layer_idx](x)
-            x = F.glu(x, dim=1)
-            x = self.dropouts[layer_idx](x)
-
-        x = x.transpose(1, 2).contiguous()  # (B, T, 908)
-        x = self.linear_layers[0](x)
-        x = F.glu(x, dim=2)
-        x = self.dropouts[-1](x)
-        x = self.linear_layers[1](x)
-
-        assert x.size(0) == B
-        assert x.size(1) == T
-
-        encoder_out = x.transpose(0, 1)  # (T, B, vocab_size)
-
-        # need to debug this -- find a simpler/elegant way in pytorch APIs
-        encoder_padding_mask = (
-            torch.arange(T).view(1, T).expand(B, -1).to(x.device)
-            >= src_lengths.view(B, 1).expand(-1, T)
-        ).t()  # (B x T) -> (T x B)
-
-        return {
-            "encoder_out": encoder_out,  # (T, B, vocab_size)
-            "encoder_padding_mask": encoder_padding_mask,  # (T, B)
-        }
-
-    def reorder_encoder_out(self, encoder_out, new_order):
-        encoder_out["encoder_out"] = encoder_out["encoder_out"].index_select(
-            1, new_order
-        )
-        encoder_out["encoder_padding_mask"] = encoder_out[
-            "encoder_padding_mask"
-        ].index_select(1, new_order)
-        return encoder_out
-
-    def max_positions(self):
-        """Maximum input length supported by the encoder."""
-        return (1e6, 1e6)  # an arbitrary large number
-
-
-@register_model_architecture("asr_w2l_conv_glu_encoder", "w2l_conv_glu_enc")
-def w2l_conv_glu_enc(args):
-    args.input_feat_per_channel = getattr(args, "input_feat_per_channel", 80)
-    args.in_channels = getattr(args, "in_channels", 1)
-    args.conv_enc_config = getattr(args, "conv_enc_config", default_conv_enc_config)

audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/new/README.md

@@ -1,43 +0,0 @@
-# Flashlight Decoder
-
-This script runs decoding for pre-trained speech recognition models.
-
-## Usage
-
-Assuming a few variables:
-
-```bash
-checkpoint=<path-to-checkpoint>
-data=<path-to-data-directory>
-lm_model=<path-to-language-model>
-lexicon=<path-to-lexicon>
-```
-
-Example usage for decoding a fine-tuned Wav2Vec model:
-
-```bash
-python $FAIRSEQ_ROOT/examples/speech_recognition/new/infer.py --multirun \
-    task=audio_pretraining \
-    task.data=$data \
-    task.labels=ltr \
-    common_eval.path=$checkpoint \
-    decoding.type=kenlm \
-    decoding.lexicon=$lexicon \
-    decoding.lmpath=$lm_model \
-    dataset.gen_subset=dev_clean,dev_other,test_clean,test_other
-```
-
-Example usage for using Ax to sweep WER parameters (requires `pip install hydra-ax-sweeper`):
-
-```bash
-python $FAIRSEQ_ROOT/examples/speech_recognition/new/infer.py --multirun \
-    hydra/sweeper=ax \
-    task=audio_pretraining \
-    task.data=$data \
-    task.labels=ltr \
-    common_eval.path=$checkpoint \
-    decoding.type=kenlm \
-    decoding.lexicon=$lexicon \
-    decoding.lmpath=$lm_model \
-    dataset.gen_subset=dev_other
-```

audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/new/conf/hydra/sweeper/ax.yaml

@@ -1,29 +0,0 @@
-# @package hydra.sweeper
-_target_: hydra_plugins.hydra_ax_sweeper.ax_sweeper.AxSweeper
-max_batch_size: null
-ax_config:
-  max_trials: 128
-  early_stop:
-    minimize: true
-    max_epochs_without_improvement: 10
-    epsilon: 0.025
-  experiment:
-    name: ${dataset.gen_subset}
-    objective_name: wer
-    minimize: true
-    parameter_constraints: null
-    outcome_constraints: null
-    status_quo: null
-  client:
-    verbose_logging: false
-    random_seed: null
-  params:
-    decoding.lmweight:
-      type: range
-      bounds: [0.0, 5.0]
-    decoding.wordscore:
-      type: range
-      bounds: [-5.0, 5.0]
-    decoding.silweight:
-      type: range
-      bounds: [ -8.0, 0.0 ]

audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/new/conf/infer.yaml

@@ -1,25 +0,0 @@
-# @package _group_
-
-defaults:
-    - task: null
-    - model: null
-
-hydra:
-  run:
-    dir: ${common_eval.results_path}/${dataset.gen_subset}
-  sweep:
-    dir: /checkpoint/${env:USER}/${env:PREFIX}/${common_eval.results_path}
-    subdir: ${dataset.gen_subset}
-common_eval:
-  results_path: null
-  path: null
-  post_process: letter
-  quiet: true
-dataset:
-  max_tokens: 3000000
-  gen_subset: test
-distributed_training:
-  distributed_world_size: 1
-decoding:
-  beam: 5
-  type: viterbi

audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/new/decoders/base_decoder.py

@@ -1,62 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-import itertools as it
-from typing import Any, Dict, List
-
-import torch
-from fairseq.data.dictionary import Dictionary
-from fairseq.models.fairseq_model import FairseqModel
-
-
-class BaseDecoder:
-    def __init__(self, tgt_dict: Dictionary) -> None:
-        self.tgt_dict = tgt_dict
-        self.vocab_size = len(tgt_dict)
-
-        self.blank = (
-            tgt_dict.index("<ctc_blank>")
-            if "<ctc_blank>" in tgt_dict.indices
-            else tgt_dict.bos()
-        )
-        if "<sep>" in tgt_dict.indices:
-            self.silence = tgt_dict.index("<sep>")
-        elif "|" in tgt_dict.indices:
-            self.silence = tgt_dict.index("|")
-        else:
-            self.silence = tgt_dict.eos()
-
-    def generate(
-        self, models: List[FairseqModel], sample: Dict[str, Any], **unused
-    ) -> List[List[Dict[str, torch.LongTensor]]]:
-        encoder_input = {
-            k: v for k, v in sample["net_input"].items() if k != "prev_output_tokens"
-        }
-        emissions = self.get_emissions(models, encoder_input)
-        return self.decode(emissions)
-
-    def get_emissions(
-        self,
-        models: List[FairseqModel],
-        encoder_input: Dict[str, Any],
-    ) -> torch.FloatTensor:
-        model = models[0]
-        encoder_out = model(**encoder_input)
-        if hasattr(model, "get_logits"):
-            emissions = model.get_logits(encoder_out)
-        else:
-            emissions = model.get_normalized_probs(encoder_out, log_probs=True)
-        return emissions.transpose(0, 1).float().cpu().contiguous()
-
-    def get_tokens(self, idxs: torch.IntTensor) -> torch.LongTensor:
-        idxs = (g[0] for g in it.groupby(idxs))
-        idxs = filter(lambda x: x != self.blank, idxs)
-        return torch.LongTensor(list(idxs))
-
-    def decode(
-        self,
-        emissions: torch.FloatTensor,
-    ) -> List[List[Dict[str, torch.LongTensor]]]:
-        raise NotImplementedError

audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/new/decoders/decoder.py

@@ -1,32 +0,0 @@
-#!/usr/bin/env python3
-
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-from typing import Union
-
-from fairseq.data.dictionary import Dictionary
-
-from .base_decoder import BaseDecoder
-from .decoder_config import DecoderConfig, FlashlightDecoderConfig
-
-
-def Decoder(
-    cfg: Union[DecoderConfig, FlashlightDecoderConfig], tgt_dict: Dictionary
-) -> BaseDecoder:
-
-    if cfg.type == "viterbi":
-        from .viterbi_decoder import ViterbiDecoder
-
-        return ViterbiDecoder(tgt_dict)
-    if cfg.type == "kenlm":
-        from .flashlight_decoder import KenLMDecoder
-
-        return KenLMDecoder(cfg, tgt_dict)
-    if cfg.type == "fairseqlm":
-        from .flashlight_decoder import FairseqLMDecoder
-
-        return FairseqLMDecoder(cfg, tgt_dict)
-    raise NotImplementedError(f"Invalid decoder name: {cfg.name}")

audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/new/decoders/decoder_config.py

@@ -1,69 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-import math
-from dataclasses import dataclass, field
-from typing import Optional
-
-from fairseq.dataclass.configs import FairseqDataclass
-from fairseq.dataclass.constants import ChoiceEnum
-from omegaconf import MISSING
-
-DECODER_CHOICES = ChoiceEnum(["viterbi", "kenlm", "fairseqlm"])
-
-
-@dataclass
-class DecoderConfig(FairseqDataclass):
-    type: DECODER_CHOICES = field(
-        default="viterbi",
-        metadata={"help": "The type of decoder to use"},
-    )
-
-
-@dataclass
-class FlashlightDecoderConfig(FairseqDataclass):
-    nbest: int = field(
-        default=1,
-        metadata={"help": "Number of decodings to return"},
-    )
-    unitlm: bool = field(
-        default=False,
-        metadata={"help": "If set, use unit language model"},
-    )
-    lmpath: str = field(
-        default=MISSING,
-        metadata={"help": "Language model for KenLM decoder"},
-    )
-    lexicon: Optional[str] = field(
-        default=None,
-        metadata={"help": "Lexicon for Flashlight decoder"},
-    )
-    beam: int = field(
-        default=50,
-        metadata={"help": "Number of beams to use for decoding"},
-    )
-    beamthreshold: float = field(
-        default=50.0,
-        metadata={"help": "Threshold for beam search decoding"},
-    )
-    beamsizetoken: Optional[int] = field(
-        default=None, metadata={"help": "Beam size to use"}
-    )
-    wordscore: float = field(
-        default=-1,
-        metadata={"help": "Word score for KenLM decoder"},
-    )
-    unkweight: float = field(
-        default=-math.inf,
-        metadata={"help": "Unknown weight for KenLM decoder"},
-    )
-    silweight: float = field(
-        default=0,
-        metadata={"help": "Silence weight for KenLM decoder"},
-    )
-    lmweight: float = field(
-        default=2,
-        metadata={"help": "Weight for LM while interpolating score"},
-    )

audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/new/decoders/flashlight_decoder.py

@@ -1,429 +0,0 @@
-#!/usr/bin/env python3
-
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-import gc
-import os.path as osp
-import warnings
-from collections import deque, namedtuple
-from typing import Any, Dict, List, Tuple
-
-import numpy as np
-import torch
-from fairseq import tasks
-from fairseq.data.dictionary import Dictionary
-from fairseq.dataclass.utils import convert_namespace_to_omegaconf
-from fairseq.models.fairseq_model import FairseqModel
-from fairseq.utils import apply_to_sample
-from omegaconf import OmegaConf, open_dict
-
-from .base_decoder import BaseDecoder
-from .decoder_config import FlashlightDecoderConfig
-
-try:
-    from flashlight.lib.text.decoder import (
-        LM,
-        CriterionType,
-        DecodeResult,
-        KenLM,
-        LexiconDecoder,
-        LexiconDecoderOptions,
-        LexiconFreeDecoder,
-        LexiconFreeDecoderOptions,
-        LMState,
-        SmearingMode,
-        Trie,
-    )
-    from flashlight.lib.text.dictionary import create_word_dict, load_words
-except ImportError:
-    warnings.warn(
-        "flashlight python bindings are required to use this functionality. "
-        "Please install from "
-        "https://github.com/facebookresearch/flashlight/tree/master/bindings/python"
-    )
-    LM = object
-    LMState = object
-
-
-class KenLMDecoder(BaseDecoder):
-    def __init__(self, cfg: FlashlightDecoderConfig, tgt_dict: Dictionary) -> None:
-        super().__init__(tgt_dict)
-
-        self.nbest = cfg.nbest
-        self.unitlm = cfg.unitlm
-
-        if cfg.lexicon:
-            self.lexicon = load_words(cfg.lexicon)
-            self.word_dict = create_word_dict(self.lexicon)
-            self.unk_word = self.word_dict.get_index("<unk>")
-
-            self.lm = KenLM(cfg.lmpath, self.word_dict)
-            self.trie = Trie(self.vocab_size, self.silence)
-
-            start_state = self.lm.start(False)
-            for word, spellings in self.lexicon.items():
-                word_idx = self.word_dict.get_index(word)
-                _, score = self.lm.score(start_state, word_idx)
-                for spelling in spellings:
-                    spelling_idxs = [tgt_dict.index(token) for token in spelling]
-                    assert (
-                        tgt_dict.unk() not in spelling_idxs
-                    ), f"{word} {spelling} {spelling_idxs}"
-                    self.trie.insert(spelling_idxs, word_idx, score)
-            self.trie.smear(SmearingMode.MAX)
-
-            self.decoder_opts = LexiconDecoderOptions(
-                beam_size=cfg.beam,
-                beam_size_token=cfg.beamsizetoken or len(tgt_dict),
-                beam_threshold=cfg.beamthreshold,
-                lm_weight=cfg.lmweight,
-                word_score=cfg.wordscore,
-                unk_score=cfg.unkweight,
-                sil_score=cfg.silweight,
-                log_add=False,
-                criterion_type=CriterionType.CTC,
-            )
-
-            self.decoder = LexiconDecoder(
-                self.decoder_opts,
-                self.trie,
-                self.lm,
-                self.silence,
-                self.blank,
-                self.unk_word,
-                [],
-                self.unitlm,
-            )
-        else:
-            assert self.unitlm, "Lexicon-free decoding requires unit LM"
-
-            d = {w: [[w]] for w in tgt_dict.symbols}
-            self.word_dict = create_word_dict(d)
-            self.lm = KenLM(cfg.lmpath, self.word_dict)
-            self.decoder_opts = LexiconFreeDecoderOptions(
-                beam_size=cfg.beam,
-                beam_size_token=cfg.beamsizetoken or len(tgt_dict),
-                beam_threshold=cfg.beamthreshold,
-                lm_weight=cfg.lmweight,
-                sil_score=cfg.silweight,
-                log_add=False,
-                criterion_type=CriterionType.CTC,
-            )
-            self.decoder = LexiconFreeDecoder(
-                self.decoder_opts, self.lm, self.silence, self.blank, []
-            )
-
-    def get_timesteps(self, token_idxs: List[int]) -> List[int]:
-        """Returns frame numbers corresponding to every non-blank token.
-
-        Parameters
-        ----------
-        token_idxs : List[int]
-            IDs of decoded tokens.
-
-        Returns
-        -------
-        List[int]
-            Frame numbers corresponding to every non-blank token.
-        """
-        timesteps = []
-        for i, token_idx in enumerate(token_idxs):
-            if token_idx == self.blank:
-                continue
-            if i == 0 or token_idx != token_idxs[i-1]:
-                timesteps.append(i)
-        return timesteps
-
-    def decode(
-        self,
-        emissions: torch.FloatTensor,
-    ) -> List[List[Dict[str, torch.LongTensor]]]:
-        B, T, N = emissions.size()
-        hypos = []
-        for b in range(B):
-            emissions_ptr = emissions.data_ptr() + 4 * b * emissions.stride(0)
-            results = self.decoder.decode(emissions_ptr, T, N)
-
-            nbest_results = results[: self.nbest]
-            hypos.append(
-                [
-                    {
-                        "tokens": self.get_tokens(result.tokens),
-                        "score": result.score,
-                        "timesteps": self.get_timesteps(result.tokens),
-                        "words": [
-                            self.word_dict.get_entry(x) for x in result.words if x >= 0
-                        ],
-                    }
-                    for result in nbest_results
-                ]
-            )
-        return hypos
-
-
-FairseqLMState = namedtuple(
-    "FairseqLMState",
-    [
-        "prefix",
-        "incremental_state",
-        "probs",
-    ],
-)
-
-
-class FairseqLM(LM):
-    def __init__(self, dictionary: Dictionary, model: FairseqModel) -> None:
-        super().__init__()
-
-        self.dictionary = dictionary
-        self.model = model
-        self.unk = self.dictionary.unk()
-
-        self.save_incremental = False  # this currently does not work properly
-        self.max_cache = 20_000
-
-        if torch.cuda.is_available():
-            model.cuda()
-        model.eval()
-        model.make_generation_fast_()
-
-        self.states = {}
-        self.stateq = deque()
-
-    def start(self, start_with_nothing: bool) -> LMState:
-        state = LMState()
-        prefix = torch.LongTensor([[self.dictionary.eos()]])
-        incremental_state = {} if self.save_incremental else None
-        with torch.no_grad():
-            res = self.model(prefix.cuda(), incremental_state=incremental_state)
-            probs = self.model.get_normalized_probs(res, log_probs=True, sample=None)
-
-        if incremental_state is not None:
-            incremental_state = apply_to_sample(lambda x: x.cpu(), incremental_state)
-        self.states[state] = FairseqLMState(
-            prefix.numpy(), incremental_state, probs[0, -1].cpu().numpy()
-        )
-        self.stateq.append(state)
-
-        return state
-
-    def score(
-        self,
-        state: LMState,
-        token_index: int,
-        no_cache: bool = False,
-    ) -> Tuple[LMState, int]:
-        """
-        Evaluate language model based on the current lm state and new word
-        Parameters:
-        -----------
-        state: current lm state
-        token_index: index of the word
-                     (can be lexicon index then you should store inside LM the
-                      mapping between indices of lexicon and lm, or lm index of a word)
-        Returns:
-        --------
-        (LMState, float): pair of (new state, score for the current word)
-        """
-        curr_state = self.states[state]
-
-        def trim_cache(targ_size: int) -> None:
-            while len(self.stateq) > targ_size:
-                rem_k = self.stateq.popleft()
-                rem_st = self.states[rem_k]
-                rem_st = FairseqLMState(rem_st.prefix, None, None)
-                self.states[rem_k] = rem_st
-
-        if curr_state.probs is None:
-            new_incremental_state = (
-                curr_state.incremental_state.copy()
-                if curr_state.incremental_state is not None
-                else None
-            )
-            with torch.no_grad():
-                if new_incremental_state is not None:
-                    new_incremental_state = apply_to_sample(
-                        lambda x: x.cuda(), new_incremental_state
-                    )
-                elif self.save_incremental:
-                    new_incremental_state = {}
-
-                res = self.model(
-                    torch.from_numpy(curr_state.prefix).cuda(),
-                    incremental_state=new_incremental_state,
-                )
-                probs = self.model.get_normalized_probs(
-                    res, log_probs=True, sample=None
-                )
-
-                if new_incremental_state is not None:
-                    new_incremental_state = apply_to_sample(
-                        lambda x: x.cpu(), new_incremental_state
-                    )
-
-                curr_state = FairseqLMState(
-                    curr_state.prefix, new_incremental_state, probs[0, -1].cpu().numpy()
-                )
-
-            if not no_cache:
-                self.states[state] = curr_state
-                self.stateq.append(state)
-
-        score = curr_state.probs[token_index].item()
-
-        trim_cache(self.max_cache)
-
-        outstate = state.child(token_index)
-        if outstate not in self.states and not no_cache:
-            prefix = np.concatenate(
-                [curr_state.prefix, torch.LongTensor([[token_index]])], -1
-            )
-            incr_state = curr_state.incremental_state
-
-            self.states[outstate] = FairseqLMState(prefix, incr_state, None)
-
-        if token_index == self.unk:
-            score = float("-inf")
-
-        return outstate, score
-
-    def finish(self, state: LMState) -> Tuple[LMState, int]:
-        """
-        Evaluate eos for language model based on the current lm state
-        Returns:
-        --------
-        (LMState, float): pair of (new state, score for the current word)
-        """
-        return self.score(state, self.dictionary.eos())
-
-    def empty_cache(self) -> None:
-        self.states = {}
-        self.stateq = deque()
-        gc.collect()
-
-
-class FairseqLMDecoder(BaseDecoder):
-    def __init__(self, cfg: FlashlightDecoderConfig, tgt_dict: Dictionary) -> None:
-        super().__init__(tgt_dict)
-
-        self.nbest = cfg.nbest
-        self.unitlm = cfg.unitlm
-
-        self.lexicon = load_words(cfg.lexicon) if cfg.lexicon else None
-        self.idx_to_wrd = {}
-
-        checkpoint = torch.load(cfg.lmpath, map_location="cpu")
-
-        if "cfg" in checkpoint and checkpoint["cfg"] is not None:
-            lm_args = checkpoint["cfg"]
-        else:
-            lm_args = convert_namespace_to_omegaconf(checkpoint["args"])
-
-        if not OmegaConf.is_dict(lm_args):
-            lm_args = OmegaConf.create(lm_args)
-
-        with open_dict(lm_args.task):
-            lm_args.task.data = osp.dirname(cfg.lmpath)
-
-        task = tasks.setup_task(lm_args.task)
-        model = task.build_model(lm_args.model)
-        model.load_state_dict(checkpoint["model"], strict=False)
-
-        self.trie = Trie(self.vocab_size, self.silence)
-
-        self.word_dict = task.dictionary
-        self.unk_word = self.word_dict.unk()
-        self.lm = FairseqLM(self.word_dict, model)
-
-        if self.lexicon:
-            start_state = self.lm.start(False)
-            for i, (word, spellings) in enumerate(self.lexicon.items()):
-                if self.unitlm:
-                    word_idx = i
-                    self.idx_to_wrd[i] = word
-                    score = 0
-                else:
-                    word_idx = self.word_dict.index(word)
-                    _, score = self.lm.score(start_state, word_idx, no_cache=True)
-
-                for spelling in spellings:
-                    spelling_idxs = [tgt_dict.index(token) for token in spelling]
-                    assert (
-                        tgt_dict.unk() not in spelling_idxs
-                    ), f"{spelling} {spelling_idxs}"
-                    self.trie.insert(spelling_idxs, word_idx, score)
-            self.trie.smear(SmearingMode.MAX)
-
-            self.decoder_opts = LexiconDecoderOptions(
-                beam_size=cfg.beam,
-                beam_size_token=cfg.beamsizetoken or len(tgt_dict),
-                beam_threshold=cfg.beamthreshold,
-                lm_weight=cfg.lmweight,
-                word_score=cfg.wordscore,
-                unk_score=cfg.unkweight,
-                sil_score=cfg.silweight,
-                log_add=False,
-                criterion_type=CriterionType.CTC,
-            )
-
-            self.decoder = LexiconDecoder(
-                self.decoder_opts,
-                self.trie,
-                self.lm,
-                self.silence,
-                self.blank,
-                self.unk_word,
-                [],
-                self.unitlm,
-            )
-        else:
-            assert self.unitlm, "Lexicon-free decoding requires unit LM"
-
-            d = {w: [[w]] for w in tgt_dict.symbols}
-            self.word_dict = create_word_dict(d)
-            self.lm = KenLM(cfg.lmpath, self.word_dict)
-            self.decoder_opts = LexiconFreeDecoderOptions(
-                beam_size=cfg.beam,
-                beam_size_token=cfg.beamsizetoken or len(tgt_dict),
-                beam_threshold=cfg.beamthreshold,
-                lm_weight=cfg.lmweight,
-                sil_score=cfg.silweight,
-                log_add=False,
-                criterion_type=CriterionType.CTC,
-            )
-            self.decoder = LexiconFreeDecoder(
-                self.decoder_opts, self.lm, self.silence, self.blank, []
-            )
-
-    def decode(
-        self,
-        emissions: torch.FloatTensor,
-    ) -> List[List[Dict[str, torch.LongTensor]]]:
-        B, T, N = emissions.size()
-        hypos = []
-
-        def make_hypo(result: DecodeResult) -> Dict[str, Any]:
-            hypo = {
-                "tokens": self.get_tokens(result.tokens),
-                "score": result.score,
-            }
-            if self.lexicon:
-                hypo["words"] = [
-                    self.idx_to_wrd[x] if self.unitlm else self.word_dict[x]
-                    for x in result.words
-                    if x >= 0
-                ]
-            return hypo
-
-        for b in range(B):
-            emissions_ptr = emissions.data_ptr() + 4 * b * emissions.stride(0)
-            results = self.decoder.decode(emissions_ptr, T, N)
-
-            nbest_results = results[: self.nbest]
-            hypos.append([make_hypo(result) for result in nbest_results])
-            self.lm.empty_cache()
-
-        return hypos

audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/new/decoders/viterbi_decoder.py

@@ -1,24 +0,0 @@
-#!/usr/bin/env python3
-
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-from typing import Dict, List
-
-import torch
-
-from .base_decoder import BaseDecoder
-
-
-class ViterbiDecoder(BaseDecoder):
-    def decode(
-        self,
-        emissions: torch.FloatTensor,
-    ) -> List[List[Dict[str, torch.LongTensor]]]:
-        def get_pred(e):
-            toks = e.argmax(dim=-1).unique_consecutive()
-            return toks[toks != self.blank]
-
-        return [[{"tokens": get_pred(x), "score": 0}] for x in emissions]

audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/new/infer.py

@@ -1,474 +0,0 @@
-#!/usr/bin/env python -u
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-import ast
-import hashlib
-import logging
-import os
-import shutil
-import sys
-from dataclasses import dataclass, field, is_dataclass
-from pathlib import Path
-from typing import Any, Dict, List, Optional, Tuple, Union
-
-import editdistance
-import hydra
-import torch
-import torch.distributed as dist
-from examples.speech_recognition.new.decoders.decoder import Decoder
-from examples.speech_recognition.new.decoders.decoder_config import (
-    DecoderConfig,
-    FlashlightDecoderConfig,
-)
-from fairseq import checkpoint_utils, distributed_utils, progress_bar, tasks, utils
-from fairseq.data.data_utils import post_process
-from fairseq.dataclass.configs import (
-    CheckpointConfig,
-    CommonConfig,
-    CommonEvalConfig,
-    DatasetConfig,
-    DistributedTrainingConfig,
-    FairseqDataclass,
-)
-from fairseq.logging.meters import StopwatchMeter, TimeMeter
-from fairseq.logging.progress_bar import BaseProgressBar
-from fairseq.models.fairseq_model import FairseqModel
-from hydra.core.config_store import ConfigStore
-from omegaconf import OmegaConf
-
-logging.root.setLevel(logging.INFO)
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
-
-config_path = Path(__file__).resolve().parent / "conf"
-
-
-@dataclass
-class DecodingConfig(DecoderConfig, FlashlightDecoderConfig):
-    unique_wer_file: bool = field(
-        default=False,
-        metadata={"help": "If set, use a unique file for storing WER"},
-    )
-    results_path: Optional[str] = field(
-        default=None,
-        metadata={
-            "help": "If set, write hypothesis and reference sentences into this directory"
-        },
-    )
-
-
-@dataclass
-class InferConfig(FairseqDataclass):
-    task: Any = None
-    decoding: DecodingConfig = DecodingConfig()
-    common: CommonConfig = CommonConfig()
-    common_eval: CommonEvalConfig = CommonEvalConfig()
-    checkpoint: CheckpointConfig = CheckpointConfig()
-    distributed_training: DistributedTrainingConfig = DistributedTrainingConfig()
-    dataset: DatasetConfig = DatasetConfig()
-    is_ax: bool = field(
-        default=False,
-        metadata={
-            "help": "if true, assumes we are using ax for tuning and returns a tuple for ax to consume"
-        },
-    )
-
-
-def reset_logging():
-    root = logging.getLogger()
-    for handler in root.handlers:
-        root.removeHandler(handler)
-    root.setLevel(os.environ.get("LOGLEVEL", "INFO").upper())
-    handler = logging.StreamHandler(sys.stdout)
-    handler.setFormatter(
-        logging.Formatter(
-            fmt="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
-            datefmt="%Y-%m-%d %H:%M:%S",
-        )
-    )
-    root.addHandler(handler)
-
-
-class InferenceProcessor:
-    cfg: InferConfig
-
-    def __init__(self, cfg: InferConfig) -> None:
-        self.cfg = cfg
-        self.task = tasks.setup_task(cfg.task)
-
-        models, saved_cfg = self.load_model_ensemble()
-        self.models = models
-        self.saved_cfg = saved_cfg
-        self.tgt_dict = self.task.target_dictionary
-
-        self.task.load_dataset(
-            self.cfg.dataset.gen_subset,
-            task_cfg=saved_cfg.task,
-        )
-        self.generator = Decoder(cfg.decoding, self.tgt_dict)
-        self.gen_timer = StopwatchMeter()
-        self.wps_meter = TimeMeter()
-        self.num_sentences = 0
-        self.total_errors = 0
-        self.total_length = 0
-
-        self.hypo_words_file = None
-        self.hypo_units_file = None
-        self.ref_words_file = None
-        self.ref_units_file = None
-
-        self.progress_bar = self.build_progress_bar()
-
-    def __enter__(self) -> "InferenceProcessor":
-        if self.cfg.decoding.results_path is not None:
-            self.hypo_words_file = self.get_res_file("hypo.word")
-            self.hypo_units_file = self.get_res_file("hypo.units")
-            self.ref_words_file = self.get_res_file("ref.word")
-            self.ref_units_file = self.get_res_file("ref.units")
-        return self
-
-    def __exit__(self, *exc) -> bool:
-        if self.cfg.decoding.results_path is not None:
-            self.hypo_words_file.close()
-            self.hypo_units_file.close()
-            self.ref_words_file.close()
-            self.ref_units_file.close()
-        return False
-
-    def __iter__(self) -> Any:
-        for sample in self.progress_bar:
-            if not self.cfg.common.cpu:
-                sample = utils.move_to_cuda(sample)
-
-            # Happens on the last batch.
-            if "net_input" not in sample:
-                continue
-            yield sample
-
-    def log(self, *args, **kwargs):
-        self.progress_bar.log(*args, **kwargs)
-
-    def print(self, *args, **kwargs):
-        self.progress_bar.print(*args, **kwargs)
-
-    def get_res_file(self, fname: str) -> None:
-        fname = os.path.join(self.cfg.decoding.results_path, fname)
-        if self.data_parallel_world_size > 1:
-            fname = f"{fname}.{self.data_parallel_rank}"
-        return open(fname, "w", buffering=1)
-
-    def merge_shards(self) -> None:
-        """Merges all shard files into shard 0, then removes shard suffix."""
-
-        shard_id = self.data_parallel_rank
-        num_shards = self.data_parallel_world_size
-
-        if self.data_parallel_world_size > 1:
-
-            def merge_shards_with_root(fname: str) -> None:
-                fname = os.path.join(self.cfg.decoding.results_path, fname)
-                logger.info("Merging %s on shard %d", fname, shard_id)
-                base_fpath = Path(f"{fname}.0")
-                with open(base_fpath, "a") as out_file:
-                    for s in range(1, num_shards):
-                        shard_fpath = Path(f"{fname}.{s}")
-                        with open(shard_fpath, "r") as in_file:
-                            for line in in_file:
-                                out_file.write(line)
-                        shard_fpath.unlink()
-                shutil.move(f"{fname}.0", fname)
-
-            dist.barrier()  # ensure all shards finished writing
-            if shard_id == (0 % num_shards):
-                merge_shards_with_root("hypo.word")
-            if shard_id == (1 % num_shards):
-                merge_shards_with_root("hypo.units")
-            if shard_id == (2 % num_shards):
-                merge_shards_with_root("ref.word")
-            if shard_id == (3 % num_shards):
-                merge_shards_with_root("ref.units")
-            dist.barrier()
-
-    def optimize_model(self, model: FairseqModel) -> None:
-        model.make_generation_fast_()
-        if self.cfg.common.fp16:
-            model.half()
-        if not self.cfg.common.cpu:
-            model.cuda()
-
-    def load_model_ensemble(self) -> Tuple[List[FairseqModel], FairseqDataclass]:
-        arg_overrides = ast.literal_eval(self.cfg.common_eval.model_overrides)
-        models, saved_cfg = checkpoint_utils.load_model_ensemble(
-            utils.split_paths(self.cfg.common_eval.path, separator="\\"),
-            arg_overrides=arg_overrides,
-            task=self.task,
-            suffix=self.cfg.checkpoint.checkpoint_suffix,
-            strict=(self.cfg.checkpoint.checkpoint_shard_count == 1),
-            num_shards=self.cfg.checkpoint.checkpoint_shard_count,
-        )
-        for model in models:
-            self.optimize_model(model)
-        return models, saved_cfg
-
-    def get_dataset_itr(self, disable_iterator_cache: bool = False) -> None:
-        return self.task.get_batch_iterator(
-            dataset=self.task.dataset(self.cfg.dataset.gen_subset),
-            max_tokens=self.cfg.dataset.max_tokens,
-            max_sentences=self.cfg.dataset.batch_size,
-            max_positions=(sys.maxsize, sys.maxsize),
-            ignore_invalid_inputs=self.cfg.dataset.skip_invalid_size_inputs_valid_test,
-            required_batch_size_multiple=self.cfg.dataset.required_batch_size_multiple,
-            seed=self.cfg.common.seed,
-            num_shards=self.data_parallel_world_size,
-            shard_id=self.data_parallel_rank,
-            num_workers=self.cfg.dataset.num_workers,
-            data_buffer_size=self.cfg.dataset.data_buffer_size,
-            disable_iterator_cache=disable_iterator_cache,
-        ).next_epoch_itr(shuffle=False)
-
-    def build_progress_bar(
-        self,
-        epoch: Optional[int] = None,
-        prefix: Optional[str] = None,
-        default_log_format: str = "tqdm",
-    ) -> BaseProgressBar:
-        return progress_bar.progress_bar(
-            iterator=self.get_dataset_itr(),
-            log_format=self.cfg.common.log_format,
-            log_interval=self.cfg.common.log_interval,
-            epoch=epoch,
-            prefix=prefix,
-            tensorboard_logdir=self.cfg.common.tensorboard_logdir,
-            default_log_format=default_log_format,
-        )
-
-    @property
-    def data_parallel_world_size(self):
-        if self.cfg.distributed_training.distributed_world_size == 1:
-            return 1
-        return distributed_utils.get_data_parallel_world_size()
-
-    @property
-    def data_parallel_rank(self):
-        if self.cfg.distributed_training.distributed_world_size == 1:
-            return 0
-        return distributed_utils.get_data_parallel_rank()
-
-    def process_sentence(
-        self,
-        sample: Dict[str, Any],
-        hypo: Dict[str, Any],
-        sid: int,
-        batch_id: int,
-    ) -> Tuple[int, int]:
-        speaker = None  # Speaker can't be parsed from dataset.
-
-        if "target_label" in sample:
-            toks = sample["target_label"]
-        else:
-            toks = sample["target"]
-        toks = toks[batch_id, :]
-
-        # Processes hypothesis.
-        hyp_pieces = self.tgt_dict.string(hypo["tokens"].int().cpu())
-        if "words" in hypo:
-            hyp_words = " ".join(hypo["words"])
-        else:
-            hyp_words = post_process(hyp_pieces, self.cfg.common_eval.post_process)
-
-        # Processes target.
-        target_tokens = utils.strip_pad(toks, self.tgt_dict.pad())
-        tgt_pieces = self.tgt_dict.string(target_tokens.int().cpu())
-        tgt_words = post_process(tgt_pieces, self.cfg.common_eval.post_process)
-
-        if self.cfg.decoding.results_path is not None:
-            print(f"{hyp_pieces} ({speaker}-{sid})", file=self.hypo_units_file)
-            print(f"{hyp_words} ({speaker}-{sid})", file=self.hypo_words_file)
-            print(f"{tgt_pieces} ({speaker}-{sid})", file=self.ref_units_file)
-            print(f"{tgt_words} ({speaker}-{sid})", file=self.ref_words_file)
-
-        if not self.cfg.common_eval.quiet:
-            logger.info(f"HYPO: {hyp_words}")
-            logger.info(f"REF: {tgt_words}")
-            logger.info("---------------------")
-
-        hyp_words, tgt_words = hyp_words.split(), tgt_words.split()
-
-        return editdistance.eval(hyp_words, tgt_words), len(tgt_words)
-
-    def process_sample(self, sample: Dict[str, Any]) -> None:
-        self.gen_timer.start()
-        hypos = self.task.inference_step(
-            generator=self.generator,
-            models=self.models,
-            sample=sample,
-        )
-        num_generated_tokens = sum(len(h[0]["tokens"]) for h in hypos)
-        self.gen_timer.stop(num_generated_tokens)
-        self.wps_meter.update(num_generated_tokens)
-
-        for batch_id, sample_id in enumerate(sample["id"].tolist()):
-            errs, length = self.process_sentence(
-                sample=sample,
-                sid=sample_id,
-                batch_id=batch_id,
-                hypo=hypos[batch_id][0],
-            )
-            self.total_errors += errs
-            self.total_length += length
-
-        self.log({"wps": round(self.wps_meter.avg)})
-        if "nsentences" in sample:
-            self.num_sentences += sample["nsentences"]
-        else:
-            self.num_sentences += sample["id"].numel()
-
-    def log_generation_time(self) -> None:
-        logger.info(
-            "Processed %d sentences (%d tokens) in %.1fs %.2f "
-            "sentences per second, %.2f tokens per second)",
-            self.num_sentences,
-            self.gen_timer.n,
-            self.gen_timer.sum,
-            self.num_sentences / (self.gen_timer.sum + 1e-6),
-            1.0 / (self.gen_timer.avg + 1e-6),
-        )
-
-
-def parse_wer(wer_file: Path) -> float:
-    with open(wer_file, "r") as f:
-        return float(f.readline().strip().split(" ")[1])
-
-
-def get_wer_file(cfg: InferConfig) -> Path:
-    """Hashes the decoding parameters to a unique file ID."""
-    base_path = "wer"
-    if cfg.decoding.results_path is not None:
-        base_path = os.path.join(cfg.decoding.results_path, base_path)
-
-    if cfg.decoding.unique_wer_file:
-        yaml_str = OmegaConf.to_yaml(cfg.decoding)
-        fid = int(hashlib.md5(yaml_str.encode("utf-8")).hexdigest(), 16)
-        return Path(f"{base_path}.{fid % 1000000}")
-    else:
-        return Path(base_path)
-
-
-def main(cfg: InferConfig) -> float:
-    """Entry point for main processing logic.
-
-    Args:
-        cfg: The inferance configuration to use.
-        wer: Optional shared memory pointer for returning the WER. If not None,
-            the final WER value will be written here instead of being returned.
-
-    Returns:
-        The final WER if `wer` is None, otherwise None.
-    """
-
-    yaml_str, wer_file = OmegaConf.to_yaml(cfg.decoding), get_wer_file(cfg)
-
-    # Validates the provided configuration.
-    if cfg.dataset.max_tokens is None and cfg.dataset.batch_size is None:
-        cfg.dataset.max_tokens = 4000000
-    if not cfg.common.cpu and not torch.cuda.is_available():
-        raise ValueError("CUDA not found; set `cpu=True` to run without CUDA")
-
-    logger.info(cfg.common_eval.path)
-
-    with InferenceProcessor(cfg) as processor:
-        for sample in processor:
-            processor.process_sample(sample)
-
-        processor.log_generation_time()
-
-        if cfg.decoding.results_path is not None:
-            processor.merge_shards()
-
-        errs_t, leng_t = processor.total_errors, processor.total_length
-
-        if cfg.common.cpu:
-            logger.warning("Merging WER requires CUDA.")
-        elif processor.data_parallel_world_size > 1:
-            stats = torch.LongTensor([errs_t, leng_t]).cuda()
-            dist.all_reduce(stats, op=dist.ReduceOp.SUM)
-            errs_t, leng_t = stats[0].item(), stats[1].item()
-
-        wer = errs_t * 100.0 / leng_t
-
-        if distributed_utils.is_master(cfg.distributed_training):
-            with open(wer_file, "w") as f:
-                f.write(
-                    (
-                        f"WER: {wer}\n"
-                        f"err / num_ref_words = {errs_t} / {leng_t}\n\n"
-                        f"{yaml_str}"
-                    )
-                )
-
-        return wer
-
-
-@hydra.main(config_path=config_path, config_name="infer")
-def hydra_main(cfg: InferConfig) -> Union[float, Tuple[float, Optional[float]]]:
-    container = OmegaConf.to_container(cfg, resolve=True, enum_to_str=True)
-    cfg = OmegaConf.create(container)
-    OmegaConf.set_struct(cfg, True)
-
-    if cfg.common.reset_logging:
-        reset_logging()
-
-    utils.import_user_module(cfg.common)
-
-    # logger.info("Config:\n%s", OmegaConf.to_yaml(cfg))
-    wer = float("inf")
-
-    try:
-        if cfg.common.profile:
-            with torch.cuda.profiler.profile():
-                with torch.autograd.profiler.emit_nvtx():
-                    distributed_utils.call_main(cfg, main)
-        else:
-            distributed_utils.call_main(cfg, main)
-
-        wer = parse_wer(get_wer_file(cfg))
-    except BaseException as e:  # pylint: disable=broad-except
-        if not cfg.common.suppress_crashes:
-            raise
-        else:
-            logger.error("Crashed! %s", str(e))
-
-    logger.info("Word error rate: %.4f", wer)
-    if cfg.is_ax:
-        return wer, None
-
-    return wer
-
-
-def cli_main() -> None:
-    try:
-        from hydra._internal.utils import (  # pylint: disable=import-outside-toplevel
-            get_args,
-        )
-
-        cfg_name = get_args().config_name or "infer"
-    except ImportError:
-        logger.warning("Failed to get config name from hydra args")
-        cfg_name = "infer"
-
-    cs = ConfigStore.instance()
-    cs.store(name=cfg_name, node=InferConfig)
-
-    for k in InferConfig.__dataclass_fields__:
-        if is_dataclass(InferConfig.__dataclass_fields__[k].type):
-            v = InferConfig.__dataclass_fields__[k].default
-            cs.store(name=k, node=v)
-
-    hydra_main()  # pylint: disable=no-value-for-parameter
-
-
-if __name__ == "__main__":
-    cli_main()

audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/tasks/__init__.py

@@ -1,7 +0,0 @@
-import importlib
-import os
-
-for file in sorted(os.listdir(os.path.dirname(__file__))):
-    if file.endswith(".py") and not file.startswith("_"):
-        task_name = file[: file.find(".py")]
-        importlib.import_module("examples.speech_recognition.tasks." + task_name)

audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/tasks/speech_recognition.py

@@ -1,157 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-import json
-import os
-import re
-import sys
-
-import torch
-from examples.speech_recognition.data import AsrDataset
-from examples.speech_recognition.data.replabels import replabel_symbol
-from fairseq.data import Dictionary
-from fairseq.tasks import LegacyFairseqTask, register_task
-
-
-def get_asr_dataset_from_json(data_json_path, tgt_dict):
-    """
-    Parse data json and create dataset.
-    See scripts/asr_prep_json.py which pack json from raw files
-
-    Json example:
-    {
-    "utts": {
-        "4771-29403-0025": {
-            "input": {
-                "length_ms": 170,
-                "path": "/tmp/file1.flac"
-            },
-            "output": {
-                "text": "HELLO \n",
-                "token": "HE LLO",
-                "tokenid": "4815, 861"
-            }
-        },
-        "1564-142299-0096": {
-            ...
-        }
-    }
-    """
-    if not os.path.isfile(data_json_path):
-        raise FileNotFoundError("Dataset not found: {}".format(data_json_path))
-    with open(data_json_path, "rb") as f:
-        data_samples = json.load(f)["utts"]
-        assert len(data_samples) != 0
-        sorted_samples = sorted(
-            data_samples.items(),
-            key=lambda sample: int(sample[1]["input"]["length_ms"]),
-            reverse=True,
-        )
-        aud_paths = [s[1]["input"]["path"] for s in sorted_samples]
-        ids = [s[0] for s in sorted_samples]
-        speakers = []
-        for s in sorted_samples:
-            m = re.search("(.+?)-(.+?)-(.+?)", s[0])
-            speakers.append(m.group(1) + "_" + m.group(2))
-        frame_sizes = [s[1]["input"]["length_ms"] for s in sorted_samples]
-        tgt = [
-            [int(i) for i in s[1]["output"]["tokenid"].split(", ")]
-            for s in sorted_samples
-        ]
-        # append eos
-        tgt = [[*t, tgt_dict.eos()] for t in tgt]
-        return AsrDataset(aud_paths, frame_sizes, tgt, tgt_dict, ids, speakers)
-
-
-@register_task("speech_recognition")
-class SpeechRecognitionTask(LegacyFairseqTask):
-    """
-    Task for training speech recognition model.
-    """
-
-    @staticmethod
-    def add_args(parser):
-        """Add task-specific arguments to the parser."""
-        parser.add_argument("data", help="path to data directory")
-        parser.add_argument(
-            "--silence-token", default="\u2581", help="token for silence (used by w2l)"
-        )
-        parser.add_argument(
-            "--max-source-positions",
-            default=sys.maxsize,
-            type=int,
-            metavar="N",
-            help="max number of frames in the source sequence",
-        )
-        parser.add_argument(
-            "--max-target-positions",
-            default=1024,
-            type=int,
-            metavar="N",
-            help="max number of tokens in the target sequence",
-        )
-
-    def __init__(self, args, tgt_dict):
-        super().__init__(args)
-        self.tgt_dict = tgt_dict
-
-    @classmethod
-    def setup_task(cls, args, **kwargs):
-        """Setup the task (e.g., load dictionaries)."""
-        dict_path = os.path.join(args.data, "dict.txt")
-        if not os.path.isfile(dict_path):
-            raise FileNotFoundError("Dict not found: {}".format(dict_path))
-        tgt_dict = Dictionary.load(dict_path)
-
-        if args.criterion == "ctc_loss":
-            tgt_dict.add_symbol("<ctc_blank>")
-        elif args.criterion == "asg_loss":
-            for i in range(1, args.max_replabel + 1):
-                tgt_dict.add_symbol(replabel_symbol(i))
-
-        print("| dictionary: {} types".format(len(tgt_dict)))
-        return cls(args, tgt_dict)
-
-    def load_dataset(self, split, combine=False, **kwargs):
-        """Load a given dataset split.
-
-        Args:
-            split (str): name of the split (e.g., train, valid, test)
-        """
-        data_json_path = os.path.join(self.args.data, "{}.json".format(split))
-        self.datasets[split] = get_asr_dataset_from_json(data_json_path, self.tgt_dict)
-
-    def build_generator(self, models, args, **unused):
-        w2l_decoder = getattr(args, "w2l_decoder", None)
-        if w2l_decoder == "viterbi":
-            from examples.speech_recognition.w2l_decoder import W2lViterbiDecoder
-
-            return W2lViterbiDecoder(args, self.target_dictionary)
-        elif w2l_decoder == "kenlm":
-            from examples.speech_recognition.w2l_decoder import W2lKenLMDecoder
-
-            return W2lKenLMDecoder(args, self.target_dictionary)
-        elif w2l_decoder == "fairseqlm":
-            from examples.speech_recognition.w2l_decoder import W2lFairseqLMDecoder
-
-            return W2lFairseqLMDecoder(args, self.target_dictionary)
-        else:
-            return super().build_generator(models, args)
-
-    @property
-    def target_dictionary(self):
-        """Return the :class:`~fairseq.data.Dictionary` for the language
-        model."""
-        return self.tgt_dict
-
-    @property
-    def source_dictionary(self):
-        """Return the source :class:`~fairseq.data.Dictionary` (if applicable
-        for this task)."""
-        return None
-
-    def max_positions(self):
-        """Return the max speech and sentence length allowed by the task."""
-        return (self.args.max_source_positions, self.args.max_target_positions)

audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/utils/wer_utils.py

@@ -1,380 +0,0 @@
-#!/usr/bin/env python3
-
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-from __future__ import absolute_import, division, print_function, unicode_literals
-
-import re
-from collections import deque
-from enum import Enum
-
-import numpy as np
-
-"""
-    Utility modules for computation of Word Error Rate,
-    Alignments, as well as more granular metrics like
-    deletion, insersion and substitutions.
-"""
-
-
-class Code(Enum):
-    match = 1
-    substitution = 2
-    insertion = 3
-    deletion = 4
-
-
-class Token(object):
-    def __init__(self, lbl="", st=np.nan, en=np.nan):
-        if np.isnan(st):
-            self.label, self.start, self.end = "", 0.0, 0.0
-        else:
-            self.label, self.start, self.end = lbl, st, en
-
-
-class AlignmentResult(object):
-    def __init__(self, refs, hyps, codes, score):
-        self.refs = refs  # std::deque<int>
-        self.hyps = hyps  # std::deque<int>
-        self.codes = codes  # std::deque<Code>
-        self.score = score  # float
-
-
-def coordinate_to_offset(row, col, ncols):
-    return int(row * ncols + col)
-
-
-def offset_to_row(offset, ncols):
-    return int(offset / ncols)
-
-
-def offset_to_col(offset, ncols):
-    return int(offset % ncols)
-
-
-def trimWhitespace(str):
-    return re.sub(" +", " ", re.sub(" *$", "", re.sub("^ *", "", str)))
-
-
-def str2toks(str):
-    pieces = trimWhitespace(str).split(" ")
-    toks = []
-    for p in pieces:
-        toks.append(Token(p, 0.0, 0.0))
-    return toks
-
-
-class EditDistance(object):
-    def __init__(self, time_mediated):
-        self.time_mediated_ = time_mediated
-        self.scores_ = np.nan  # Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic>
-        self.backtraces_ = (
-            np.nan
-        )  # Eigen::Matrix<size_t, Eigen::Dynamic, Eigen::Dynamic> backtraces_;
-        self.confusion_pairs_ = {}
-
-    def cost(self, ref, hyp, code):
-        if self.time_mediated_:
-            if code == Code.match:
-                return abs(ref.start - hyp.start) + abs(ref.end - hyp.end)
-            elif code == Code.insertion:
-                return hyp.end - hyp.start
-            elif code == Code.deletion:
-                return ref.end - ref.start
-            else:  # substitution
-                return abs(ref.start - hyp.start) + abs(ref.end - hyp.end) + 0.1
-        else:
-            if code == Code.match:
-                return 0
-            elif code == Code.insertion or code == Code.deletion:
-                return 3
-            else:  # substitution
-                return 4
-
-    def get_result(self, refs, hyps):
-        res = AlignmentResult(refs=deque(), hyps=deque(), codes=deque(), score=np.nan)
-
-        num_rows, num_cols = self.scores_.shape
-        res.score = self.scores_[num_rows - 1, num_cols - 1]
-
-        curr_offset = coordinate_to_offset(num_rows - 1, num_cols - 1, num_cols)
-
-        while curr_offset != 0:
-            curr_row = offset_to_row(curr_offset, num_cols)
-            curr_col = offset_to_col(curr_offset, num_cols)
-
-            prev_offset = self.backtraces_[curr_row, curr_col]
-
-            prev_row = offset_to_row(prev_offset, num_cols)
-            prev_col = offset_to_col(prev_offset, num_cols)
-
-            res.refs.appendleft(curr_row - 1)  # Note: this was .push_front() in C++
-            res.hyps.appendleft(curr_col - 1)
-            if curr_row - 1 == prev_row and curr_col == prev_col:
-                res.codes.appendleft(Code.deletion)
-            elif curr_row == prev_row and curr_col - 1 == prev_col:
-                res.codes.appendleft(Code.insertion)
-            else:
-                # assert(curr_row - 1 == prev_row and curr_col - 1 == prev_col)
-                ref_str = refs[res.refs[0]].label
-                hyp_str = hyps[res.hyps[0]].label
-
-                if ref_str == hyp_str:
-                    res.codes.appendleft(Code.match)
-                else:
-                    res.codes.appendleft(Code.substitution)
-
-                    confusion_pair = "%s -> %s" % (ref_str, hyp_str)
-                    if confusion_pair not in self.confusion_pairs_:
-                        self.confusion_pairs_[confusion_pair] = 1
-                    else:
-                        self.confusion_pairs_[confusion_pair] += 1
-
-            curr_offset = prev_offset
-
-        return res
-
-    def align(self, refs, hyps):
-        if len(refs) == 0 and len(hyps) == 0:
-            return np.nan
-
-        # NOTE: we're not resetting the values in these matrices because every value
-        # will be overridden in the loop below. If this assumption doesn't hold,
-        # be sure to set all entries in self.scores_ and self.backtraces_ to 0.
-        self.scores_ = np.zeros((len(refs) + 1, len(hyps) + 1))
-        self.backtraces_ = np.zeros((len(refs) + 1, len(hyps) + 1))
-
-        num_rows, num_cols = self.scores_.shape
-
-        for i in range(num_rows):
-            for j in range(num_cols):
-                if i == 0 and j == 0:
-                    self.scores_[i, j] = 0.0
-                    self.backtraces_[i, j] = 0
-                    continue
-
-                if i == 0:
-                    self.scores_[i, j] = self.scores_[i, j - 1] + self.cost(
-                        None, hyps[j - 1], Code.insertion
-                    )
-                    self.backtraces_[i, j] = coordinate_to_offset(i, j - 1, num_cols)
-                    continue
-
-                if j == 0:
-                    self.scores_[i, j] = self.scores_[i - 1, j] + self.cost(
-                        refs[i - 1], None, Code.deletion
-                    )
-                    self.backtraces_[i, j] = coordinate_to_offset(i - 1, j, num_cols)
-                    continue
-
-                # Below here both i and j are greater than 0
-                ref = refs[i - 1]
-                hyp = hyps[j - 1]
-                best_score = self.scores_[i - 1, j - 1] + (
-                    self.cost(ref, hyp, Code.match)
-                    if (ref.label == hyp.label)
-                    else self.cost(ref, hyp, Code.substitution)
-                )
-
-                prev_row = i - 1
-                prev_col = j - 1
-                ins = self.scores_[i, j - 1] + self.cost(None, hyp, Code.insertion)
-                if ins < best_score:
-                    best_score = ins
-                    prev_row = i
-                    prev_col = j - 1
-
-                delt = self.scores_[i - 1, j] + self.cost(ref, None, Code.deletion)
-                if delt < best_score:
-                    best_score = delt
-                    prev_row = i - 1
-                    prev_col = j
-
-                self.scores_[i, j] = best_score
-                self.backtraces_[i, j] = coordinate_to_offset(
-                    prev_row, prev_col, num_cols
-                )
-
-        return self.get_result(refs, hyps)
-
-
-class WERTransformer(object):
-    def __init__(self, hyp_str, ref_str, verbose=True):
-        self.ed_ = EditDistance(False)
-        self.id2oracle_errs_ = {}
-        self.utts_ = 0
-        self.words_ = 0
-        self.insertions_ = 0
-        self.deletions_ = 0
-        self.substitutions_ = 0
-
-        self.process(["dummy_str", hyp_str, ref_str])
-
-        if verbose:
-            print("'%s' vs '%s'" % (hyp_str, ref_str))
-            self.report_result()
-
-    def process(self, input):  # std::vector<std::string>&& input
-        if len(input) < 3:
-            print(
-                "Input must be of the form <id> ... <hypo> <ref> , got ",
-                len(input),
-                " inputs:",
-            )
-            return None
-
-        # Align
-        # std::vector<Token> hyps;
-        # std::vector<Token> refs;
-
-        hyps = str2toks(input[-2])
-        refs = str2toks(input[-1])
-
-        alignment = self.ed_.align(refs, hyps)
-        if alignment is None:
-            print("Alignment is null")
-            return np.nan
-
-        # Tally errors
-        ins = 0
-        dels = 0
-        subs = 0
-        for code in alignment.codes:
-            if code == Code.substitution:
-                subs += 1
-            elif code == Code.insertion:
-                ins += 1
-            elif code == Code.deletion:
-                dels += 1
-
-        # Output
-        row = input
-        row.append(str(len(refs)))
-        row.append(str(ins))
-        row.append(str(dels))
-        row.append(str(subs))
-        # print(row)
-
-        # Accumulate
-        kIdIndex = 0
-        kNBestSep = "/"
-
-        pieces = input[kIdIndex].split(kNBestSep)
-
-        if len(pieces) == 0:
-            print(
-                "Error splitting ",
-                input[kIdIndex],
-                " on '",
-                kNBestSep,
-                "', got empty list",
-            )
-            return np.nan
-
-        id = pieces[0]
-        if id not in self.id2oracle_errs_:
-            self.utts_ += 1
-            self.words_ += len(refs)
-            self.insertions_ += ins
-            self.deletions_ += dels
-            self.substitutions_ += subs
-            self.id2oracle_errs_[id] = [ins, dels, subs]
-        else:
-            curr_err = ins + dels + subs
-            prev_err = np.sum(self.id2oracle_errs_[id])
-            if curr_err < prev_err:
-                self.id2oracle_errs_[id] = [ins, dels, subs]
-
-        return 0
-
-    def report_result(self):
-        # print("----------  Summary ---------------")
-        if self.words_ == 0:
-            print("No words counted")
-            return
-
-        # 1-best
-        best_wer = (
-            100.0
-            * (self.insertions_ + self.deletions_ + self.substitutions_)
-            / self.words_
-        )
-
-        print(
-            "\tWER = %0.2f%% (%i utts, %i words, %0.2f%% ins, "
-            "%0.2f%% dels, %0.2f%% subs)"
-            % (
-                best_wer,
-                self.utts_,
-                self.words_,
-                100.0 * self.insertions_ / self.words_,
-                100.0 * self.deletions_ / self.words_,
-                100.0 * self.substitutions_ / self.words_,
-            )
-        )
-
-    def wer(self):
-        if self.words_ == 0:
-            wer = np.nan
-        else:
-            wer = (
-                100.0
-                * (self.insertions_ + self.deletions_ + self.substitutions_)
-                / self.words_
-            )
-        return wer
-
-    def stats(self):
-        if self.words_ == 0:
-            stats = {}
-        else:
-            wer = (
-                100.0
-                * (self.insertions_ + self.deletions_ + self.substitutions_)
-                / self.words_
-            )
-            stats = dict(
-                {
-                    "wer": wer,
-                    "utts": self.utts_,
-                    "numwords": self.words_,
-                    "ins": self.insertions_,
-                    "dels": self.deletions_,
-                    "subs": self.substitutions_,
-                    "confusion_pairs": self.ed_.confusion_pairs_,
-                }
-            )
-        return stats
-
-
-def calc_wer(hyp_str, ref_str):
-    t = WERTransformer(hyp_str, ref_str, verbose=0)
-    return t.wer()
-
-
-def calc_wer_stats(hyp_str, ref_str):
-    t = WERTransformer(hyp_str, ref_str, verbose=0)
-    return t.stats()
-
-
-def get_wer_alignment_codes(hyp_str, ref_str):
-    """
-    INPUT: hypothesis string, reference string
-    OUTPUT: List of alignment codes (intermediate results from WER computation)
-    """
-    t = WERTransformer(hyp_str, ref_str, verbose=0)
-    return t.ed_.align(str2toks(ref_str), str2toks(hyp_str)).codes
-
-
-def merge_counts(x, y):
-    # Merge two hashes which have 'counts' as their values
-    # This can be used for example to merge confusion pair counts
-    #   conf_pairs = merge_counts(conf_pairs, stats['confusion_pairs'])
-    for k, v in y.items():
-        if k not in x:
-            x[k] = 0
-        x[k] += v
-    return x

audio/safeear/safeear_repo/fairseq_ours/examples/speech_recognition/w2l_decoder.py

@@ -1,491 +0,0 @@
-#!/usr/bin/env python3
-
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-"""
-Flashlight decoders.
-"""
-
-import gc
-import itertools as it
-import os.path as osp
-import warnings
-from collections import deque, namedtuple
-from typing import List
-
-import numpy as np
-import torch
-from examples.speech_recognition.data.replabels import unpack_replabels
-from fairseq import tasks
-from fairseq.dataclass.utils import convert_namespace_to_omegaconf
-from fairseq.utils import apply_to_sample
-from omegaconf import open_dict
-
-try:
-    from flashlight.lib.sequence.criterion import CpuViterbiPath, get_data_ptr_as_bytes
-    from flashlight.lib.text.decoder import (
-        LM,
-        CriterionType,
-        KenLM,
-        LexiconDecoder,
-        LexiconDecoderOptions,
-        LMState,
-        SmearingMode,
-        Trie,
-    )
-    from flashlight.lib.text.dictionary import create_word_dict, load_words
-except:
-    warnings.warn(
-        "flashlight python bindings are required to use this functionality. Please install from https://github.com/facebookresearch/flashlight/tree/master/bindings/python"
-    )
-    LM = object
-    LMState = object
-
-
-class W2lDecoder(object):
-    def __init__(self, args, tgt_dict):
-        self.tgt_dict = tgt_dict
-        self.vocab_size = len(tgt_dict)
-        self.nbest = args.nbest
-
-        # criterion-specific init
-        self.criterion_type = CriterionType.CTC
-        self.blank = (
-            tgt_dict.index("<ctc_blank>")
-            if "<ctc_blank>" in tgt_dict.indices
-            else tgt_dict.bos()
-        )
-        if "<sep>" in tgt_dict.indices:
-            self.silence = tgt_dict.index("<sep>")
-        elif "|" in tgt_dict.indices:
-            self.silence = tgt_dict.index("|")
-        else:
-            self.silence = tgt_dict.eos()
-        self.asg_transitions = None
-
-    def generate(self, models, sample, **unused):
-        """Generate a batch of inferences."""
-        # model.forward normally channels prev_output_tokens into the decoder
-        # separately, but SequenceGenerator directly calls model.encoder
-        encoder_input = {
-            k: v for k, v in sample["net_input"].items() if k != "prev_output_tokens"
-        }
-        emissions = self.get_emissions(models, encoder_input)
-        return self.decode(emissions)
-
-    def get_emissions(self, models, encoder_input):
-        """Run encoder and normalize emissions"""
-        model = models[0]
-        encoder_out = model(**encoder_input)
-        if hasattr(model, "get_logits"):
-            emissions = model.get_logits(encoder_out) # no need to normalize emissions
-        else:
-            emissions = model.get_normalized_probs(encoder_out, log_probs=True)
-        return emissions.transpose(0, 1).float().cpu().contiguous()
-
-    def get_tokens(self, idxs):
-        """Normalize tokens by handling CTC blank, ASG replabels, etc."""
-        idxs = (g[0] for g in it.groupby(idxs))
-        idxs = filter(lambda x: x != self.blank, idxs)
-        return torch.LongTensor(list(idxs))
-
-
-class W2lViterbiDecoder(W2lDecoder):
-    def __init__(self, args, tgt_dict):
-        super().__init__(args, tgt_dict)
-
-    def decode(self, emissions):
-        B, T, N = emissions.size()
-        hypos = []
-        if self.asg_transitions is None:
-            transitions = torch.FloatTensor(N, N).zero_()
-        else:
-            transitions = torch.FloatTensor(self.asg_transitions).view(N, N)
-        viterbi_path = torch.IntTensor(B, T)
-        workspace = torch.ByteTensor(CpuViterbiPath.get_workspace_size(B, T, N))
-        CpuViterbiPath.compute(
-            B,
-            T,
-            N,
-            get_data_ptr_as_bytes(emissions),
-            get_data_ptr_as_bytes(transitions),
-            get_data_ptr_as_bytes(viterbi_path),
-            get_data_ptr_as_bytes(workspace),
-        )
-        return [
-            [{"tokens": self.get_tokens(viterbi_path[b].tolist()), "score": 0}]
-            for b in range(B)
-        ]
-
-
-class W2lKenLMDecoder(W2lDecoder):
-    def __init__(self, args, tgt_dict):
-        super().__init__(args, tgt_dict)
-
-        self.unit_lm = getattr(args, "unit_lm", False)
-
-        if args.lexicon:
-            self.lexicon = load_words(args.lexicon)
-            self.word_dict = create_word_dict(self.lexicon)
-            self.unk_word = self.word_dict.get_index("<unk>")
-
-            self.lm = KenLM(args.kenlm_model, self.word_dict)
-            self.trie = Trie(self.vocab_size, self.silence)
-
-            start_state = self.lm.start(False)
-            for i, (word, spellings) in enumerate(self.lexicon.items()):
-                word_idx = self.word_dict.get_index(word)
-                _, score = self.lm.score(start_state, word_idx)
-                for spelling in spellings:
-                    spelling_idxs = [tgt_dict.index(token) for token in spelling]
-                    assert (
-                        tgt_dict.unk() not in spelling_idxs
-                    ), f"{spelling} {spelling_idxs}"
-                    self.trie.insert(spelling_idxs, word_idx, score)
-            self.trie.smear(SmearingMode.MAX)
-
-            self.decoder_opts = LexiconDecoderOptions(
-                beam_size=args.beam,
-                beam_size_token=int(getattr(args, "beam_size_token", len(tgt_dict))),
-                beam_threshold=args.beam_threshold,
-                lm_weight=args.lm_weight,
-                word_score=args.word_score,
-                unk_score=args.unk_weight,
-                sil_score=args.sil_weight,
-                log_add=False,
-                criterion_type=self.criterion_type,
-            )
-
-            if self.asg_transitions is None:
-                N = 768
-                # self.asg_transitions = torch.FloatTensor(N, N).zero_()
-                self.asg_transitions = []
-
-            self.decoder = LexiconDecoder(
-                self.decoder_opts,
-                self.trie,
-                self.lm,
-                self.silence,
-                self.blank,
-                self.unk_word,
-                self.asg_transitions,
-                self.unit_lm,
-            )
-        else:
-            assert args.unit_lm, "lexicon free decoding can only be done with a unit language model"
-            from flashlight.lib.text.decoder import (
-                LexiconFreeDecoder,
-                LexiconFreeDecoderOptions,
-            )
-
-            d = {w: [[w]] for w in tgt_dict.symbols}
-            self.word_dict = create_word_dict(d)
-            self.lm = KenLM(args.kenlm_model, self.word_dict)
-            self.decoder_opts = LexiconFreeDecoderOptions(
-                beam_size=args.beam,
-                beam_size_token=int(getattr(args, "beam_size_token", len(tgt_dict))),
-                beam_threshold=args.beam_threshold,
-                lm_weight=args.lm_weight,
-                sil_score=args.sil_weight,
-                log_add=False,
-                criterion_type=self.criterion_type,
-            )
-            self.decoder = LexiconFreeDecoder(
-                self.decoder_opts, self.lm, self.silence, self.blank, []
-            )
-
-    def get_timesteps(self, token_idxs: List[int]) -> List[int]:
-        """Returns frame numbers corresponding to every non-blank token.
-
-        Parameters
-        ----------
-        token_idxs : List[int]
-            IDs of decoded tokens.
-
-        Returns
-        -------
-        List[int]
-            Frame numbers corresponding to every non-blank token.
-        """
-        timesteps = []
-        for i, token_idx in enumerate(token_idxs):
-            if token_idx == self.blank:
-                continue
-            if i == 0 or token_idx != token_idxs[i-1]:
-                timesteps.append(i)
-        return timesteps
-
-    def decode(self, emissions):
-        B, T, N = emissions.size()
-        hypos = []
-        for b in range(B):
-            emissions_ptr = emissions.data_ptr() + 4 * b * emissions.stride(0)
-            results = self.decoder.decode(emissions_ptr, T, N)
-
-            nbest_results = results[: self.nbest]
-            hypos.append(
-                [
-                    {
-                        "tokens": self.get_tokens(result.tokens),
-                        "score": result.score,
-                        "timesteps": self.get_timesteps(result.tokens),
-                        "words": [
-                            self.word_dict.get_entry(x) for x in result.words if x >= 0
-                        ],
-                    }
-                    for result in nbest_results
-                ]
-            )
-        return hypos
-
-
-FairseqLMState = namedtuple("FairseqLMState", ["prefix", "incremental_state", "probs"])
-
-
-class FairseqLM(LM):
-    def __init__(self, dictionary, model):
-        LM.__init__(self)
-        self.dictionary = dictionary
-        self.model = model
-        self.unk = self.dictionary.unk()
-
-        self.save_incremental = False  # this currently does not work properly
-        self.max_cache = 20_000
-
-        model.cuda()
-        model.eval()
-        model.make_generation_fast_()
-
-        self.states = {}
-        self.stateq = deque()
-
-    def start(self, start_with_nothing):
-        state = LMState()
-        prefix = torch.LongTensor([[self.dictionary.eos()]])
-        incremental_state = {} if self.save_incremental else None
-        with torch.no_grad():
-            res = self.model(prefix.cuda(), incremental_state=incremental_state)
-            probs = self.model.get_normalized_probs(res, log_probs=True, sample=None)
-
-        if incremental_state is not None:
-            incremental_state = apply_to_sample(lambda x: x.cpu(), incremental_state)
-        self.states[state] = FairseqLMState(
-            prefix.numpy(), incremental_state, probs[0, -1].cpu().numpy()
-        )
-        self.stateq.append(state)
-
-        return state
-
-    def score(self, state: LMState, token_index: int, no_cache: bool = False):
-        """
-        Evaluate language model based on the current lm state and new word
-        Parameters:
-        -----------
-        state: current lm state
-        token_index: index of the word
-                     (can be lexicon index then you should store inside LM the
-                      mapping between indices of lexicon and lm, or lm index of a word)
-
-        Returns:
-        --------
-        (LMState, float): pair of (new state, score for the current word)
-        """
-        curr_state = self.states[state]
-
-        def trim_cache(targ_size):
-            while len(self.stateq) > targ_size:
-                rem_k = self.stateq.popleft()
-                rem_st = self.states[rem_k]
-                rem_st = FairseqLMState(rem_st.prefix, None, None)
-                self.states[rem_k] = rem_st
-
-        if curr_state.probs is None:
-            new_incremental_state = (
-                curr_state.incremental_state.copy()
-                if curr_state.incremental_state is not None
-                else None
-            )
-            with torch.no_grad():
-                if new_incremental_state is not None:
-                    new_incremental_state = apply_to_sample(
-                        lambda x: x.cuda(), new_incremental_state
-                    )
-                elif self.save_incremental:
-                    new_incremental_state = {}
-
-                res = self.model(
-                    torch.from_numpy(curr_state.prefix).cuda(),
-                    incremental_state=new_incremental_state,
-                )
-                probs = self.model.get_normalized_probs(
-                    res, log_probs=True, sample=None
-                )
-
-                if new_incremental_state is not None:
-                    new_incremental_state = apply_to_sample(
-                        lambda x: x.cpu(), new_incremental_state
-                    )
-
-                curr_state = FairseqLMState(
-                    curr_state.prefix, new_incremental_state, probs[0, -1].cpu().numpy()
-                )
-
-            if not no_cache:
-                self.states[state] = curr_state
-                self.stateq.append(state)
-
-        score = curr_state.probs[token_index].item()
-
-        trim_cache(self.max_cache)
-
-        outstate = state.child(token_index)
-        if outstate not in self.states and not no_cache:
-            prefix = np.concatenate(
-                [curr_state.prefix, torch.LongTensor([[token_index]])], -1
-            )
-            incr_state = curr_state.incremental_state
-
-            self.states[outstate] = FairseqLMState(prefix, incr_state, None)
-
-        if token_index == self.unk:
-            score = float("-inf")
-
-        return outstate, score
-
-    def finish(self, state: LMState):
-        """
-        Evaluate eos for language model based on the current lm state
-
-        Returns:
-        --------
-        (LMState, float): pair of (new state, score for the current word)
-        """
-        return self.score(state, self.dictionary.eos())
-
-    def empty_cache(self):
-        self.states = {}
-        self.stateq = deque()
-        gc.collect()
-
-
-class W2lFairseqLMDecoder(W2lDecoder):
-    def __init__(self, args, tgt_dict):
-        super().__init__(args, tgt_dict)
-
-        self.unit_lm = getattr(args, "unit_lm", False)
-
-        self.lexicon = load_words(args.lexicon) if args.lexicon else None
-        self.idx_to_wrd = {}
-
-        checkpoint = torch.load(args.kenlm_model, map_location="cpu")
-
-        if "cfg" in checkpoint and checkpoint["cfg"] is not None:
-            lm_args = checkpoint["cfg"]
-        else:
-            lm_args = convert_namespace_to_omegaconf(checkpoint["args"])
-
-        with open_dict(lm_args.task):
-            lm_args.task.data = osp.dirname(args.kenlm_model)
-
-        task = tasks.setup_task(lm_args.task)
-        model = task.build_model(lm_args.model)
-        model.load_state_dict(checkpoint["model"], strict=False)
-
-        self.trie = Trie(self.vocab_size, self.silence)
-
-        self.word_dict = task.dictionary
-        self.unk_word = self.word_dict.unk()
-        self.lm = FairseqLM(self.word_dict, model)
-
-        if self.lexicon:
-            start_state = self.lm.start(False)
-            for i, (word, spellings) in enumerate(self.lexicon.items()):
-                if self.unit_lm:
-                    word_idx = i
-                    self.idx_to_wrd[i] = word
-                    score = 0
-                else:
-                    word_idx = self.word_dict.index(word)
-                    _, score = self.lm.score(start_state, word_idx, no_cache=True)
-
-                for spelling in spellings:
-                    spelling_idxs = [tgt_dict.index(token) for token in spelling]
-                    assert (
-                        tgt_dict.unk() not in spelling_idxs
-                    ), f"{spelling} {spelling_idxs}"
-                    self.trie.insert(spelling_idxs, word_idx, score)
-            self.trie.smear(SmearingMode.MAX)
-
-            self.decoder_opts = LexiconDecoderOptions(
-                beam_size=args.beam,
-                beam_size_token=int(getattr(args, "beam_size_token", len(tgt_dict))),
-                beam_threshold=args.beam_threshold,
-                lm_weight=args.lm_weight,
-                word_score=args.word_score,
-                unk_score=args.unk_weight,
-                sil_score=args.sil_weight,
-                log_add=False,
-                criterion_type=self.criterion_type,
-            )
-
-            self.decoder = LexiconDecoder(
-                self.decoder_opts,
-                self.trie,
-                self.lm,
-                self.silence,
-                self.blank,
-                self.unk_word,
-                [],
-                self.unit_lm,
-            )
-        else:
-            assert args.unit_lm, "lexicon free decoding can only be done with a unit language model"
-            from flashlight.lib.text.decoder import (
-                LexiconFreeDecoder,
-                LexiconFreeDecoderOptions,
-            )
-
-            d = {w: [[w]] for w in tgt_dict.symbols}
-            self.word_dict = create_word_dict(d)
-            self.lm = KenLM(args.kenlm_model, self.word_dict)
-            self.decoder_opts = LexiconFreeDecoderOptions(
-                beam_size=args.beam,
-                beam_size_token=int(getattr(args, "beam_size_token", len(tgt_dict))),
-                beam_threshold=args.beam_threshold,
-                lm_weight=args.lm_weight,
-                sil_score=args.sil_weight,
-                log_add=False,
-                criterion_type=self.criterion_type,
-            )
-            self.decoder = LexiconFreeDecoder(
-                self.decoder_opts, self.lm, self.silence, self.blank, []
-            )
-
-    def decode(self, emissions):
-        B, T, N = emissions.size()
-        hypos = []
-
-        def idx_to_word(idx):
-            if self.unit_lm:
-                return self.idx_to_wrd[idx]
-            else:
-                return self.word_dict[idx]
-
-        def make_hypo(result):
-            hypo = {"tokens": self.get_tokens(result.tokens), "score": result.score}
-            if self.lexicon:
-                hypo["words"] = [idx_to_word(x) for x in result.words if x >= 0]
-            return hypo
-
-        for b in range(B):
-            emissions_ptr = emissions.data_ptr() + 4 * b * emissions.stride(0)
-            results = self.decoder.decode(emissions_ptr, T, N)
-
-            nbest_results = results[: self.nbest]
-            hypos.append([make_hypo(result) for result in nbest_results])
-            self.lm.empty_cache()
-
-        return hypos

audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/README.md

@@ -1,38 +0,0 @@
-Speech Synthesis (S^2)
-===
-[https://arxiv.org/abs/2109.06912](https://arxiv.org/abs/2109.06912)
-
-Speech synthesis with fairseq.
-
-## Features
-
-- Autoregressive and non-autoregressive models
-- Multi-speaker synthesis
-- Audio preprocessing (denoising, VAD, etc.) for less curated data
-- Automatic metrics for model development
-- Similar data configuration as [S2T](../speech_to_text/README.md)
-
-
-## Examples
-- [Single-speaker synthesis on LJSpeech](docs/ljspeech_example.md)
-- [Multi-speaker synthesis on VCTK](docs/vctk_example.md)
-- [Multi-speaker synthesis on Common Voice](docs/common_voice_example.md)
-
-
-## Citation
-Please cite as:
-```
-@article{wang2021fairseqs2,
-  title={fairseq S\^{} 2: A Scalable and Integrable Speech Synthesis Toolkit},
-  author={Wang, Changhan and Hsu, Wei-Ning and Adi, Yossi and Polyak, Adam and Lee, Ann and Chen, Peng-Jen and Gu, Jiatao and Pino, Juan},
-  journal={arXiv preprint arXiv:2109.06912},
-  year={2021}
-}
-
-@inproceedings{ott2019fairseq,
-  title = {fairseq: A Fast, Extensible Toolkit for Sequence Modeling},
-  author = {Myle Ott and Sergey Edunov and Alexei Baevski and Angela Fan and Sam Gross and Nathan Ng and David Grangier and Michael Auli},
-  booktitle = {Proceedings of NAACL-HLT 2019: Demonstrations},
-  year = {2019},
-}
-```

audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/__init__.py

@@ -1,4 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.

audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/data_utils.py

@@ -1,347 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-import io
-import os
-import tempfile
-import zipfile
-from dataclasses import dataclass
-from itertools import groupby
-from pathlib import Path
-from typing import Dict, List, Optional
-
-import numpy as np
-import torch
-import torch.nn.functional as F
-from examples.speech_to_text.data_utils import load_tsv_to_dicts
-from fairseq.data.audio.audio_utils import (
-    TTSMelScale,
-    TTSSpectrogram,
-    is_npy_data,
-    parse_path,
-    read_from_stored_zip,
-)
-from tqdm import tqdm
-
-
-def trim_or_pad_to_target_length(
-        data_1d_or_2d: np.ndarray, target_length: int
-) -> np.ndarray:
-    assert len(data_1d_or_2d.shape) in {1, 2}
-    delta = data_1d_or_2d.shape[0] - target_length
-    if delta >= 0:  # trim if being longer
-        data_1d_or_2d = data_1d_or_2d[: target_length]
-    else:  # pad if being shorter
-        if len(data_1d_or_2d.shape) == 1:
-            data_1d_or_2d = np.concatenate(
-                [data_1d_or_2d, np.zeros(-delta)], axis=0
-            )
-        else:
-            data_1d_or_2d = np.concatenate(
-                [data_1d_or_2d, np.zeros((-delta, data_1d_or_2d.shape[1]))],
-                axis=0
-            )
-    return data_1d_or_2d
-
-
-def extract_logmel_spectrogram(
-        waveform: torch.Tensor, sample_rate: int,
-        output_path: Optional[Path] = None, win_length: int = 1024,
-        hop_length: int = 256, n_fft: int = 1024,
-        win_fn: callable = torch.hann_window, n_mels: int = 80,
-        f_min: float = 0., f_max: float = 8000, eps: float = 1e-5,
-        overwrite: bool = False, target_length: Optional[int] = None
-):
-    if output_path is not None and output_path.is_file() and not overwrite:
-        return
-
-    spectrogram_transform = TTSSpectrogram(
-        n_fft=n_fft, win_length=win_length, hop_length=hop_length,
-        window_fn=win_fn
-    )
-    mel_scale_transform = TTSMelScale(
-        n_mels=n_mels, sample_rate=sample_rate, f_min=f_min, f_max=f_max,
-        n_stft=n_fft // 2 + 1
-    )
-    spectrogram = spectrogram_transform(waveform)
-    mel_spec = mel_scale_transform(spectrogram)
-    logmel_spec = torch.clamp(mel_spec, min=eps).log()
-    assert len(logmel_spec.shape) == 3 and logmel_spec.shape[0] == 1
-    logmel_spec = logmel_spec.squeeze().t()  # D x T -> T x D
-    if target_length is not None:
-        logmel_spec = trim_or_pad_to_target_length(logmel_spec, target_length)
-
-    if output_path is not None:
-        np.save(output_path.as_posix(), logmel_spec)
-    else:
-        return logmel_spec
-
-
-def extract_pitch(
-        waveform: torch.Tensor, sample_rate: int,
-        output_path: Optional[Path] = None, hop_length: int = 256,
-        log_scale: bool = True, phoneme_durations: Optional[List[int]] = None
-):
-    if output_path is not None and output_path.is_file():
-        return
-
-    try:
-        import pyworld
-    except ImportError:
-        raise ImportError("Please install PyWORLD: pip install pyworld")
-
-    _waveform = waveform.squeeze(0).double().numpy()
-    pitch, t = pyworld.dio(
-        _waveform, sample_rate, frame_period=hop_length / sample_rate * 1000
-    )
-    pitch = pyworld.stonemask(_waveform, pitch, t, sample_rate)
-
-    if phoneme_durations is not None:
-        pitch = trim_or_pad_to_target_length(pitch, sum(phoneme_durations))
-        try:
-            from scipy.interpolate import interp1d
-        except ImportError:
-            raise ImportError("Please install SciPy: pip install scipy")
-        nonzero_ids = np.where(pitch != 0)[0]
-        if len(nonzero_ids) == 0:
-            print((f"{output_path} has all empty values in the pitch contour"))
-            return
-        elif len(nonzero_ids) == 1:
-            print((f"{output_path} has only one non-zero values in the pitch contour"))
-            return
-        else:
-            interp_fn = interp1d(
-                nonzero_ids,
-                pitch[nonzero_ids],
-                fill_value=(pitch[nonzero_ids[0]], pitch[nonzero_ids[-1]]),
-                bounds_error=False,
-            )
-            pitch = interp_fn(np.arange(0, len(pitch)))
-        d_cumsum = np.cumsum(np.concatenate([np.array([0]), phoneme_durations]))
-        pitch = np.array(
-            [
-                np.mean(pitch[d_cumsum[i-1]: d_cumsum[i]])
-                for i in range(1, len(d_cumsum))
-            ]
-        )
-        assert len(pitch) == len(phoneme_durations)
-
-    if log_scale:
-        pitch = np.log(pitch + 1)
-
-    if output_path is not None:
-        np.save(output_path.as_posix(), pitch)
-    else:
-        return pitch
-
-
-def extract_energy(
-        waveform: torch.Tensor, output_path: Optional[Path] = None,
-        hop_length: int = 256, n_fft: int = 1024, log_scale: bool = True,
-        phoneme_durations: Optional[List[int]] = None
-):
-    if output_path is not None and output_path.is_file():
-        return
-
-    assert len(waveform.shape) == 2 and waveform.shape[0] == 1
-    waveform = waveform.view(1, 1, waveform.shape[1])
-    waveform = F.pad(
-        waveform.unsqueeze(1), [n_fft // 2, n_fft // 2, 0, 0],
-        mode="reflect"
-    )
-    waveform = waveform.squeeze(1)
-
-    fourier_basis = np.fft.fft(np.eye(n_fft))
-    cutoff = int((n_fft / 2 + 1))
-    fourier_basis = np.vstack(
-        [np.real(fourier_basis[:cutoff, :]),
-         np.imag(fourier_basis[:cutoff, :])]
-    )
-
-    forward_basis = torch.FloatTensor(fourier_basis[:, None, :])
-    forward_transform = F.conv1d(
-        waveform, forward_basis, stride=hop_length, padding=0
-    )
-
-    real_part = forward_transform[:, :cutoff, :]
-    imag_part = forward_transform[:, cutoff:, :]
-    magnitude = torch.sqrt(real_part ** 2 + imag_part ** 2)
-    energy = torch.norm(magnitude, dim=1).squeeze(0).numpy()
-
-    if phoneme_durations is not None:
-        energy = trim_or_pad_to_target_length(energy, sum(phoneme_durations))
-        d_cumsum = np.cumsum(np.concatenate([np.array([0]), phoneme_durations]))
-        energy = np.array(
-            [
-                np.mean(energy[d_cumsum[i - 1]: d_cumsum[i]])
-                for i in range(1, len(d_cumsum))
-            ]
-        )
-        assert len(energy) == len(phoneme_durations)
-
-    if log_scale:
-        energy = np.log(energy + 1)
-
-    if output_path is not None:
-        np.save(output_path.as_posix(), energy)
-    else:
-        return energy
-
-
-def get_global_cmvn(feature_root: Path, output_path: Optional[Path] = None):
-    mean_x, mean_x2, n_frames = None, None, 0
-    feature_paths = feature_root.glob("*.npy")
-    for p in tqdm(feature_paths):
-        with open(p, 'rb') as f:
-            frames = np.load(f).squeeze()
-
-        n_frames += frames.shape[0]
-
-        cur_mean_x = frames.sum(axis=0)
-        if mean_x is None:
-            mean_x = cur_mean_x
-        else:
-            mean_x += cur_mean_x
-
-        cur_mean_x2 = (frames ** 2).sum(axis=0)
-        if mean_x2 is None:
-            mean_x2 = cur_mean_x2
-        else:
-            mean_x2 += cur_mean_x2
-
-    mean_x /= n_frames
-    mean_x2 /= n_frames
-    var_x = mean_x2 - mean_x ** 2
-    std_x = np.sqrt(np.maximum(var_x, 1e-10))
-
-    if output_path is not None:
-        with open(output_path, 'wb') as f:
-            np.savez(f, mean=mean_x, std=std_x)
-    else:
-        return {"mean": mean_x, "std": std_x}
-
-
-def ipa_phonemize(text, lang="en-us", use_g2p=False):
-    if use_g2p:
-        assert lang == "en-us", "g2pE phonemizer only works for en-us"
-        try:
-            from g2p_en import G2p
-            g2p = G2p()
-            return " ".join("|" if p == " " else p for p in g2p(text))
-        except ImportError:
-            raise ImportError(
-                "Please install phonemizer: pip install g2p_en"
-            )
-    else:
-        try:
-            from phonemizer import phonemize
-            from phonemizer.separator import Separator
-            return phonemize(
-                text, backend='espeak', language=lang,
-                separator=Separator(word="| ", phone=" ")
-            )
-        except ImportError:
-            raise ImportError(
-                "Please install phonemizer: pip install phonemizer"
-            )
-
-
-@dataclass
-class ForceAlignmentInfo(object):
-    tokens: List[str]
-    frame_durations: List[int]
-    start_sec: Optional[float]
-    end_sec: Optional[float]
-
-
-def get_mfa_alignment_by_sample_id(
-        textgrid_zip_path: str, sample_id: str, sample_rate: int,
-        hop_length: int, silence_phones: List[str] = ("sil", "sp", "spn")
-) -> ForceAlignmentInfo:
-    try:
-        import tgt
-    except ImportError:
-        raise ImportError("Please install TextGridTools: pip install tgt")
-
-    filename = f"{sample_id}.TextGrid"
-    out_root = Path(tempfile.gettempdir())
-    tgt_path = out_root / filename
-    with zipfile.ZipFile(textgrid_zip_path) as f_zip:
-        f_zip.extract(filename, path=out_root)
-    textgrid = tgt.io.read_textgrid(tgt_path.as_posix())
-    os.remove(tgt_path)
-
-    phones, frame_durations = [], []
-    start_sec, end_sec, end_idx = 0, 0, 0
-    for t in textgrid.get_tier_by_name("phones")._objects:
-        s, e, p = t.start_time, t.end_time, t.text
-        # Trim leading silences
-        if len(phones) == 0:
-            if p in silence_phones:
-                continue
-            else:
-                start_sec = s
-        phones.append(p)
-        if p not in silence_phones:
-            end_sec = e
-            end_idx = len(phones)
-        r = sample_rate / hop_length
-        frame_durations.append(int(np.round(e * r) - np.round(s * r)))
-    # Trim tailing silences
-    phones = phones[:end_idx]
-    frame_durations = frame_durations[:end_idx]
-
-    return ForceAlignmentInfo(
-        tokens=phones, frame_durations=frame_durations, start_sec=start_sec,
-        end_sec=end_sec
-    )
-
-
-def get_mfa_alignment(
-        textgrid_zip_path: str, sample_ids: List[str], sample_rate: int,
-        hop_length: int
-) -> Dict[str, ForceAlignmentInfo]:
-    return {
-        i: get_mfa_alignment_by_sample_id(
-            textgrid_zip_path, i, sample_rate, hop_length
-        ) for i in tqdm(sample_ids)
-    }
-
-
-def get_unit_alignment(
-        id_to_unit_tsv_path: str, sample_ids: List[str]
-) -> Dict[str, ForceAlignmentInfo]:
-    id_to_units = {
-        e["id"]: e["units"] for e in load_tsv_to_dicts(id_to_unit_tsv_path)
-    }
-    id_to_units = {i: id_to_units[i].split() for i in sample_ids}
-    id_to_units_collapsed = {
-        i: [uu for uu, _ in groupby(u)] for i, u in id_to_units.items()
-    }
-    id_to_durations = {
-        i: [len(list(g)) for _, g in groupby(u)] for i, u in id_to_units.items()
-    }
-
-    return {
-        i: ForceAlignmentInfo(
-            tokens=id_to_units_collapsed[i], frame_durations=id_to_durations[i],
-            start_sec=None, end_sec=None
-        )
-        for i in sample_ids
-    }
-
-
-def get_feature_value_min_max(feature_paths: List[str]):
-    v_min, v_max = 1e-8, -1e-8
-    for p in tqdm(feature_paths):
-        _path, slice_ptr = parse_path(p)
-        assert len(slice_ptr) == 2
-        byte_data = read_from_stored_zip(_path, slice_ptr[0], slice_ptr[1])
-        assert is_npy_data(byte_data)
-        path_or_fp = io.BytesIO(byte_data)
-        features = np.load(path_or_fp).squeeze()
-        v_min = min(v_min, features.min().item())
-        v_max = max(v_max, features.max().item())
-    return v_min, v_max

audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/docs/common_voice_example.md

@@ -1,67 +0,0 @@
-[[Back]](..)
-
-# Common Voice
-
-[Common Voice](https://commonvoice.mozilla.org/en/datasets) is a public domain speech corpus with 11.2K hours of read
-speech in 76 languages (the latest version 7.0). We provide examples for building
-[Transformer](https://arxiv.org/abs/1809.08895) models on this dataset.
-
-
-## Data preparation
-[Download](https://commonvoice.mozilla.org/en/datasets) and unpack Common Voice v4 to a path `${DATA_ROOT}/${LANG_ID}`.
-Create splits and generate audio manifests with
-```bash
-python -m examples.speech_synthesis.preprocessing.get_common_voice_audio_manifest \
-  --data-root ${DATA_ROOT} \
-  --lang ${LANG_ID} \
-  --output-manifest-root ${AUDIO_MANIFEST_ROOT} --convert-to-wav
-```
-
-To denoise audio and trim leading/trailing silence using signal processing based VAD, run
-```bash
-for SPLIT in dev test train; do
-    python -m examples.speech_synthesis.preprocessing.denoise_and_vad_audio \
-      --audio-manifest ${AUDIO_MANIFEST_ROOT}/${SPLIT}.audio.tsv \
-      --output-dir ${PROCESSED_DATA_ROOT} \
-      --denoise --vad --vad-agg-level 2
-done
-```
-
-which generates a new audio TSV manifest under `${PROCESSED_DATA_ROOT}` with updated path to the processed audio and
-a new column for SNR.
-
-To do filtering by CER, follow the [Automatic Evaluation](../docs/ljspeech_example.md#automatic-evaluation) section to
-run ASR model (add `--eval-target` to `get_eval_manifest` for evaluation on the reference audio; add `--err-unit char`
-to `eval_asr` to compute CER instead of WER). The example-level CER is saved to
-`${EVAL_OUTPUT_ROOT}/uer_cer.${SPLIT}.tsv`.
-
-Then, extract log-Mel spectrograms, generate feature manifest and create data configuration YAML with
-```bash
-python -m examples.speech_synthesis.preprocessing.get_feature_manifest \
-  --audio-manifest-root ${AUDIO_MANIFEST_ROOT} \
-  --output-root ${FEATURE_MANIFEST_ROOT} \
-  --ipa-vocab --lang ${LANG_ID} \
-  --snr-threshold 15 \
-  --cer-threshold 0.1 --cer-tsv-path ${EVAL_OUTPUT_ROOT}/uer_cer.${SPLIT}.tsv
-```
-where we use phoneme inputs (`--ipa-vocab`) as example. For sample filtering, we set the SNR and CER threshold
-to 15 and 10%, respectively.
-
-
-## Training
-(Please refer to [the LJSpeech example](../docs/ljspeech_example.md#transformer).)
-
-
-## Inference
-(Please refer to [the LJSpeech example](../docs/ljspeech_example.md#inference).)
-
-## Automatic Evaluation
-(Please refer to [the LJSpeech example](../docs/ljspeech_example.md#automatic-evaluation).)
-
-## Results
-
-| Language | Speakers | --arch | Params | Test MCD | Model |
-|---|---|---|---|---|---|
-| English | 200 | tts_transformer | 54M | 3.8 | [Download](https://dl.fbaipublicfiles.com/fairseq/s2/cv4_en200_transformer_phn.tar) |
-
-[[Back]](..)

audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/docs/ljspeech_example.md

@@ -1,137 +0,0 @@
-[[Back]](..)
-
-# LJSpeech
-
-[LJSpeech](https://keithito.com/LJ-Speech-Dataset) is a public domain TTS
-corpus with around 24 hours of English speech sampled at 22.05kHz. We provide examples for building
-[Transformer](https://arxiv.org/abs/1809.08895) and [FastSpeech 2](https://arxiv.org/abs/2006.04558)
-models on this dataset.
-
-
-## Data preparation
-
-Download data, create splits and generate audio manifests with
-```bash
-python -m examples.speech_synthesis.preprocessing.get_ljspeech_audio_manifest \
-  --output-data-root ${AUDIO_DATA_ROOT} \
-  --output-manifest-root ${AUDIO_MANIFEST_ROOT}
-```
-
-Then, extract log-Mel spectrograms, generate feature manifest and create data configuration YAML with
-```bash
-python -m examples.speech_synthesis.preprocessing.get_feature_manifest \
-  --audio-manifest-root ${AUDIO_MANIFEST_ROOT} \
-  --output-root ${FEATURE_MANIFEST_ROOT} \
-  --ipa-vocab --use-g2p
-```
-where we use phoneme inputs (`--ipa-vocab --use-g2p`) as example.
-
-FastSpeech 2 additionally requires frame durations, pitch and energy as auxiliary training targets.
-Add `--add-fastspeech-targets` to include these fields in the feature manifests. We get frame durations either from
-phoneme-level force-alignment or frame-level pseudo-text unit sequence. They should be pre-computed and specified via:
-- `--textgrid-zip ${TEXT_GRID_ZIP_PATH}` for a ZIP file, inside which there is one
-  [TextGrid](https://www.fon.hum.uva.nl/praat/manual/TextGrid.html) file per sample to provide force-alignment info.
-- `--id-to-units-tsv ${ID_TO_UNIT_TSV}` for a TSV file, where there are 2 columns for sample ID and
-  space-delimited pseudo-text unit sequence, respectively.
-
-For your convenience, we provide pre-computed
-[force-alignment](https://dl.fbaipublicfiles.com/fairseq/s2/ljspeech_mfa.zip) from
-[Montreal Forced Aligner](https://github.com/MontrealCorpusTools/Montreal-Forced-Aligner) and
-[pseudo-text units](s3://dl.fbaipublicfiles.com/fairseq/s2/ljspeech_hubert.tsv) from
-[HuBERT](https://github.com/pytorch/fairseq/tree/main/examples/hubert). You can also generate them by yourself using
-a different software or model.
-
-
-## Training
-#### Transformer
-```bash
-fairseq-train ${FEATURE_MANIFEST_ROOT} --save-dir ${SAVE_DIR} \
-  --config-yaml config.yaml --train-subset train --valid-subset dev \
-  --num-workers 4 --max-tokens 30000 --max-update 200000 \
-  --task text_to_speech --criterion tacotron2 --arch tts_transformer \
-  --clip-norm 5.0 --n-frames-per-step 4 --bce-pos-weight 5.0 \
-  --dropout 0.1 --attention-dropout 0.1 --activation-dropout 0.1 \
-  --encoder-normalize-before --decoder-normalize-before \
-  --optimizer adam --lr 2e-3 --lr-scheduler inverse_sqrt --warmup-updates 4000 \
-  --seed 1 --update-freq 8 --eval-inference --best-checkpoint-metric mcd_loss
-```
-where `SAVE_DIR` is the checkpoint root path. We set `--update-freq 8` to simulate 8 GPUs with 1 GPU. You may want to
-update it accordingly when using more than 1 GPU.
-
-#### FastSpeech2
-```bash
-fairseq-train ${FEATURE_MANIFEST_ROOT} --save-dir ${SAVE_DIR} \
-  --config-yaml config.yaml --train-subset train --valid-subset dev \
-  --num-workers 4 --max-sentences 6 --max-update 200000 \
-  --task text_to_speech --criterion fastspeech2 --arch fastspeech2 \
-  --clip-norm 5.0 --n-frames-per-step 1 \
-  --dropout 0.1 --attention-dropout 0.1 \
-  --optimizer adam --lr 5e-4 --lr-scheduler inverse_sqrt --warmup-updates 4000 \
-  --seed 1 --update-freq 8 --eval-inference --best-checkpoint-metric mcd_loss
-```
-
-
-## Inference
-Average the last 5 checkpoints, generate the test split spectrogram and waveform using the default Griffin-Lim vocoder:
-```bash
-SPLIT=test
-CHECKPOINT_NAME=avg_last_5
-CHECKPOINT_PATH=${SAVE_DIR}/checkpoint_${CHECKPOINT_NAME}.pt
-python scripts/average_checkpoints.py --inputs ${SAVE_DIR} \
-  --num-epoch-checkpoints 5 \
-  --output ${CHECKPOINT_PATH}
-
-python -m examples.speech_synthesis.generate_waveform ${FEATURE_MANIFEST_ROOT} \
-  --config-yaml config.yaml --gen-subset ${SPLIT} --task text_to_speech \
-  --path ${CHECKPOINT_PATH} --max-tokens 50000 --spec-bwd-max-iter 32 \
-  --dump-waveforms
-```
-which dumps files (waveform, feature, attention plot, etc.) to `${SAVE_DIR}/generate-${CHECKPOINT_NAME}-${SPLIT}`. To
-re-synthesize target waveforms for automatic evaluation, add `--dump-target`.
-
-## Automatic Evaluation
-To start with, generate the manifest for synthetic speech, which will be taken as inputs by evaluation scripts.
-```bash
-python -m examples.speech_synthesis.evaluation.get_eval_manifest \
-  --generation-root ${SAVE_DIR}/generate-${CHECKPOINT_NAME}-${SPLIT} \
-  --audio-manifest ${AUDIO_MANIFEST_ROOT}/${SPLIT}.audio.tsv \
-  --output-path ${EVAL_OUTPUT_ROOT}/eval.tsv \
-  --vocoder griffin_lim --sample-rate 22050 --audio-format flac \
-  --use-resynthesized-target
-```
-Speech recognition (ASR) models usually operate at lower sample rates (e.g. 16kHz). For the WER/CER metric,
-you may need to resample the audios accordingly --- add `--output-sample-rate 16000` for `generate_waveform.py` and
-use `--sample-rate 16000` for `get_eval_manifest.py`.
-
-
-#### WER/CER metric
-We use wav2vec 2.0 ASR model as example. [Download](https://github.com/pytorch/fairseq/tree/main/examples/wav2vec)
-the model checkpoint and dictionary, then compute WER/CER with
-```bash
-python -m examples.speech_synthesis.evaluation.eval_asr \
-  --audio-header syn --text-header text --err-unit char --split ${SPLIT} \
-  --w2v-ckpt ${WAV2VEC2_CHECKPOINT_PATH} --w2v-dict-dir ${WAV2VEC2_DICT_DIR} \
-  --raw-manifest ${EVAL_OUTPUT_ROOT}/eval_16khz.tsv --asr-dir ${EVAL_OUTPUT_ROOT}/asr
-```
-
-#### MCD/MSD metric
-```bash
-python -m examples.speech_synthesis.evaluation.eval_sp \
-  ${EVAL_OUTPUT_ROOT}/eval.tsv --mcd --msd
-```
-
-#### F0 metrics
-```bash
-python -m examples.speech_synthesis.evaluation.eval_f0 \
-  ${EVAL_OUTPUT_ROOT}/eval.tsv --gpe --vde --ffe
-```
-
-
-## Results
-
-| --arch | Params | Test MCD | Model |
-|---|---|---|---|
-| tts_transformer | 54M | 3.8 | [Download](https://dl.fbaipublicfiles.com/fairseq/s2/ljspeech_transformer_phn.tar) |
-| fastspeech2 | 41M | 3.8 | [Download](https://dl.fbaipublicfiles.com/fairseq/s2/ljspeech_fastspeech2_phn.tar) |
-
-[[Back]](..)

audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/docs/vctk_example.md

@@ -1,61 +0,0 @@
-[[Back]](..)
-
-# VCTK
-
-[VCTK](https://datashare.ed.ac.uk/handle/10283/3443) is an open English speech corpus. We provide examples
-for building [Transformer](https://arxiv.org/abs/1809.08895) models on this dataset.
-
-
-## Data preparation
-Download data, create splits and generate audio manifests with
-```bash
-python -m examples.speech_synthesis.preprocessing.get_vctk_audio_manifest \
-  --output-data-root ${AUDIO_DATA_ROOT} \
-  --output-manifest-root ${AUDIO_MANIFEST_ROOT}
-```
-
-To denoise audio and trim leading/trailing silence using signal processing based VAD, run
-```bash
-for SPLIT in dev test train; do
-    python -m examples.speech_synthesis.preprocessing.denoise_and_vad_audio \
-      --audio-manifest ${AUDIO_MANIFEST_ROOT}/${SPLIT}.audio.tsv \
-      --output-dir ${PROCESSED_DATA_ROOT} \
-      --denoise --vad --vad-agg-level 3
-done
-```
-which generates a new audio TSV manifest under `${PROCESSED_DATA_ROOT}` with updated path to the processed audio and
-a new column for SNR.
-
-To do filtering by CER, follow the [Automatic Evaluation](../docs/ljspeech_example.md#automatic-evaluation) section to
-run ASR model (add `--eval-target` to `get_eval_manifest` for evaluation on the reference audio; add `--err-unit char`
-to `eval_asr` to compute CER instead of WER). The example-level CER is saved to
-`${EVAL_OUTPUT_ROOT}/uer_cer.${SPLIT}.tsv`.
-
-Then, extract log-Mel spectrograms, generate feature manifest and create data configuration YAML with
-```bash
-python -m examples.speech_synthesis.preprocessing.get_feature_manifest \
-  --audio-manifest-root ${PROCESSED_DATA_ROOT} \
-  --output-root ${FEATURE_MANIFEST_ROOT} \
-  --ipa-vocab --use-g2p \
-  --snr-threshold 15 \
-  --cer-threshold 0.1 --cer-tsv-path ${EVAL_OUTPUT_ROOT}/uer_cer.${SPLIT}.tsv
-```
-where we use phoneme inputs (`--ipa-vocab --use-g2p`) as example. For sample filtering, we set the SNR and CER threshold
-to 15 and 10%, respectively.
-
-## Training
-(Please refer to [the LJSpeech example](../docs/ljspeech_example.md#transformer).)
-
-## Inference
-(Please refer to [the LJSpeech example](../docs/ljspeech_example.md#inference).)
-
-## Automatic Evaluation
-(Please refer to [the LJSpeech example](../docs/ljspeech_example.md#automatic-evaluation).)
-
-## Results
-
-| --arch | Params | Test MCD | Model |
-|---|---|---|---|
-| tts_transformer | 54M | 3.4 | [Download](https://dl.fbaipublicfiles.com/fairseq/s2/vctk_transformer_phn.tar) |
-
-[[Back]](..)

audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/evaluation/__init__.py

@@ -1,4 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.

audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/evaluation/eval_asr.py

@@ -1,128 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-import argparse
-import re
-import shutil
-import subprocess
-from pathlib import Path
-
-import editdistance
-import soundfile as sf
-from examples.speech_to_text.data_utils import load_tsv_to_dicts
-
-
-def preprocess_text(text):
-    text = "|".join(re.sub(r"[^A-Z' ]", " ", text.upper()).split())
-    text = " ".join(text)
-    return text
-
-
-def prepare_w2v_data(
-        dict_dir, sample_rate, label, audio_paths, texts, split, data_dir
-):
-    data_dir.mkdir(parents=True, exist_ok=True)
-    shutil.copyfile(
-        dict_dir / f"dict.{label}.txt",
-        data_dir / f"dict.{label}.txt"
-    )
-    with open(data_dir / f"{split}.tsv", "w") as f:
-        f.write("/\n")
-        for audio_path in audio_paths:
-            wav, sr = sf.read(audio_path)
-            assert sr == sample_rate, f"{sr} != sample_rate"
-            nsample = len(wav)
-            f.write(f"{audio_path}\t{nsample}\n")
-    with open(data_dir / f"{split}.{label}", "w") as f:
-        for text in texts:
-            text = preprocess_text(text)
-            f.write(f"{text}\n")
-
-
-def run_asr(asr_dir, split, w2v_ckpt, w2v_label, res_dir):
-    """
-    results will be saved at
-    {res_dir}/{ref,hypo}.word-{w2v_ckpt.filename}-{split}.txt
-    """
-    cmd = ["python", "-m", "examples.speech_recognition.infer"]
-    cmd += [str(asr_dir.resolve())]
-    cmd += ["--task", "audio_finetuning", "--nbest", "1", "--quiet"]
-    cmd += ["--w2l-decoder", "viterbi", "--criterion", "ctc"]
-    cmd += ["--post-process", "letter", "--max-tokens", "4000000"]
-    cmd += ["--path", str(w2v_ckpt.resolve()), "--labels", w2v_label]
-    cmd += ["--gen-subset", split, "--results-path", str(res_dir.resolve())]
-
-    print(f"running cmd:\n{' '.join(cmd)}")
-    subprocess.run(cmd, check=True)
-
-
-def compute_error_rate(hyp_wrd_path, ref_wrd_path, unit="word"):
-    """each line is "<text> (None-<index>)" """
-    tokenize_line = {
-        "word": lambda x: re.sub(r" \(.*\)$", "", x.rstrip()).split(),
-        "char": lambda x: list(re.sub(r" \(.*\)$", "", x.rstrip()))
-    }.get(unit)
-    if tokenize_line is None:
-        raise ValueError(f"{unit} not supported")
-
-    inds = [int(re.sub(r"\D*(\d*)\D*", r"\1", line))
-            for line in open(hyp_wrd_path)]
-    hyps = [tokenize_line(line) for line in open(hyp_wrd_path)]
-    refs = [tokenize_line(line) for line in open(ref_wrd_path)]
-    assert(len(hyps) == len(refs))
-    err_rates = [
-        editdistance.eval(hyp, ref) / len(ref) for hyp, ref in zip(hyps, refs)
-    ]
-    ind_to_err_rates = {i: e for i, e in zip(inds, err_rates)}
-    return ind_to_err_rates
-
-
-def main(args):
-    samples = load_tsv_to_dicts(args.raw_manifest)
-    ids = [
-        sample[args.id_header] if args.id_header else "" for sample in samples
-    ]
-    audio_paths = [sample[args.audio_header] for sample in samples]
-    texts = [sample[args.text_header] for sample in samples]
-
-    prepare_w2v_data(
-        args.w2v_dict_dir,
-        args.w2v_sample_rate,
-        args.w2v_label,
-        audio_paths,
-        texts,
-        args.split,
-        args.asr_dir
-    )
-    run_asr(args.asr_dir, args.split, args.w2v_ckpt, args.w2v_label, args.asr_dir)
-    ind_to_err_rates = compute_error_rate(
-        args.asr_dir / f"hypo.word-{args.w2v_ckpt.name}-{args.split}.txt",
-        args.asr_dir / f"ref.word-{args.w2v_ckpt.name}-{args.split}.txt",
-        args.err_unit,
-    )
-
-    uer_path = args.asr_dir / f"uer_{args.err_unit}.{args.split}.tsv"
-    with open(uer_path, "w") as f:
-        f.write("id\taudio\tuer\n")
-        for ind, (id_, audio_path) in enumerate(zip(ids, audio_paths)):
-            f.write(f"{id_}\t{audio_path}\t{ind_to_err_rates[ind]:.4f}\n")
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--raw-manifest", required=True, type=Path)
-    parser.add_argument("--asr-dir", required=True, type=Path)
-    parser.add_argument("--id-header", default="id", type=str)
-    parser.add_argument("--audio-header", default="audio", type=str)
-    parser.add_argument("--text-header", default="src_text", type=str)
-    parser.add_argument("--split", default="raw", type=str)
-    parser.add_argument("--w2v-ckpt", required=True, type=Path)
-    parser.add_argument("--w2v-dict-dir", required=True, type=Path)
-    parser.add_argument("--w2v-sample-rate", default=16000, type=int)
-    parser.add_argument("--w2v-label", default="ltr", type=str)
-    parser.add_argument("--err-unit", default="word", type=str)
-    args = parser.parse_args()
-
-    main(args)

audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/evaluation/eval_f0.py

@@ -1,267 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-"""
-Signal processing-based evaluation using waveforms
-"""
-import os.path as op
-
-import numpy as np
-import torchaudio
-import tqdm
-from examples.speech_synthesis.evaluation.eval_sp import load_eval_spec
-from examples.speech_synthesis.utils import (
-    f0_frame_error,
-    gross_pitch_error,
-    voicing_decision_error,
-)
-from tabulate import tabulate
-
-
-def difference_function(x, n, tau_max):
-    """
-    Compute difference function of data x. This solution is implemented directly
-    with Numpy fft.
-
-
-    :param x: audio data
-    :param n: length of data
-    :param tau_max: integration window size
-    :return: difference function
-    :rtype: list
-    """
-
-    x = np.array(x, np.float64)
-    w = x.size
-    tau_max = min(tau_max, w)
-    x_cumsum = np.concatenate((np.array([0.]), (x * x).cumsum()))
-    size = w + tau_max
-    p2 = (size // 32).bit_length()
-    nice_numbers = (16, 18, 20, 24, 25, 27, 30, 32)
-    size_pad = min(x * 2 ** p2 for x in nice_numbers if x * 2 ** p2 >= size)
-    fc = np.fft.rfft(x, size_pad)
-    conv = np.fft.irfft(fc * fc.conjugate())[:tau_max]
-    return x_cumsum[w:w - tau_max:-1] + x_cumsum[w] - x_cumsum[:tau_max] - \
-        2 * conv
-
-
-def cumulative_mean_normalized_difference_function(df, n):
-    """
-    Compute cumulative mean normalized difference function (CMND).
-
-    :param df: Difference function
-    :param n: length of data
-    :return: cumulative mean normalized difference function
-    :rtype: list
-    """
-
-    # scipy method
-    cmn_df = df[1:] * range(1, n) / np.cumsum(df[1:]).astype(float)
-    return np.insert(cmn_df, 0, 1)
-
-
-def get_pitch(cmdf, tau_min, tau_max, harmo_th=0.1):
-    """
-    Return fundamental period of a frame based on CMND function.
-
-    :param cmdf: Cumulative Mean Normalized Difference function
-    :param tau_min: minimum period for speech
-    :param tau_max: maximum period for speech
-    :param harmo_th: harmonicity threshold to determine if it is necessary to
-    compute pitch frequency
-    :return: fundamental period if there is values under threshold, 0 otherwise
-    :rtype: float
-    """
-    tau = tau_min
-    while tau < tau_max:
-        if cmdf[tau] < harmo_th:
-            while tau + 1 < tau_max and cmdf[tau + 1] < cmdf[tau]:
-                tau += 1
-            return tau
-        tau += 1
-
-    return 0    # if unvoiced
-
-
-def compute_yin(sig, sr, w_len=512, w_step=256, f0_min=100, f0_max=500,
-                harmo_thresh=0.1):
-    """
-
-    Compute the Yin Algorithm. Return fundamental frequency and harmonic rate.
-
-    https://github.com/NVIDIA/mellotron adaption of
-    https://github.com/patriceguyot/Yin
-
-    :param sig: Audio signal (list of float)
-    :param sr: sampling rate (int)
-    :param w_len: size of the analysis window (samples)
-    :param w_step: size of the lag between two consecutives windows (samples)
-    :param f0_min: Minimum fundamental frequency that can be detected (hertz)
-    :param f0_max: Maximum fundamental frequency that can be detected (hertz)
-    :param harmo_thresh: Threshold of detection. The yalgorithmù return the
-    first minimum of the CMND function below this threshold.
-
-    :returns:
-
-        * pitches: list of fundamental frequencies,
-        * harmonic_rates: list of harmonic rate values for each fundamental
-        frequency value (= confidence value)
-        * argmins: minimums of the Cumulative Mean Normalized DifferenceFunction
-        * times: list of time of each estimation
-    :rtype: tuple
-    """
-
-    tau_min = int(sr / f0_max)
-    tau_max = int(sr / f0_min)
-
-    # time values for each analysis window
-    time_scale = range(0, len(sig) - w_len, w_step)
-    times = [t/float(sr) for t in time_scale]
-    frames = [sig[t:t + w_len] for t in time_scale]
-
-    pitches = [0.0] * len(time_scale)
-    harmonic_rates = [0.0] * len(time_scale)
-    argmins = [0.0] * len(time_scale)
-
-    for i, frame in enumerate(frames):
-        # Compute YIN
-        df = difference_function(frame, w_len, tau_max)
-        cm_df = cumulative_mean_normalized_difference_function(df, tau_max)
-        p = get_pitch(cm_df, tau_min, tau_max, harmo_thresh)
-
-        # Get results
-        if np.argmin(cm_df) > tau_min:
-            argmins[i] = float(sr / np.argmin(cm_df))
-        if p != 0:  # A pitch was found
-            pitches[i] = float(sr / p)
-            harmonic_rates[i] = cm_df[p]
-        else:  # No pitch, but we compute a value of the harmonic rate
-            harmonic_rates[i] = min(cm_df)
-
-    return pitches, harmonic_rates, argmins, times
-
-
-def extract_f0(samples):
-    f0_samples = []
-    for sample in tqdm.tqdm(samples):
-        if not op.isfile(sample["ref"]) or not op.isfile(sample["syn"]):
-            f0_samples.append(None)
-            continue
-
-        # assume single channel
-        yref, sr = torchaudio.load(sample["ref"])
-        ysyn, _sr = torchaudio.load(sample["syn"])
-        yref, ysyn = yref[0], ysyn[0]
-        assert sr == _sr, f"{sr} != {_sr}"
-
-        yref_f0 = compute_yin(yref, sr)
-        ysyn_f0 = compute_yin(ysyn, sr)
-
-        f0_samples += [
-            {
-                "ref": yref_f0,
-                "syn": ysyn_f0
-            }
-        ]
-
-    return f0_samples
-
-
-def eval_f0_error(samples, distortion_fn):
-    results = []
-    for sample in tqdm.tqdm(samples):
-        if sample is None:
-            results.append(None)
-            continue
-        # assume single channel
-        yref_f, _, _, yref_t = sample["ref"]
-        ysyn_f, _, _, ysyn_t = sample["syn"]
-
-        yref_f = np.array(yref_f)
-        yref_t = np.array(yref_t)
-        ysyn_f = np.array(ysyn_f)
-        ysyn_t = np.array(ysyn_t)
-
-        distortion = distortion_fn(yref_t, yref_f, ysyn_t, ysyn_f)
-        results.append((distortion.item(),
-                        len(yref_f),
-                        len(ysyn_f)
-                        ))
-    return results
-
-
-def eval_gross_pitch_error(samples):
-    return eval_f0_error(samples, gross_pitch_error)
-
-
-def eval_voicing_decision_error(samples):
-    return eval_f0_error(samples, voicing_decision_error)
-
-
-def eval_f0_frame_error(samples):
-    return eval_f0_error(samples, f0_frame_error)
-
-
-def print_results(results, show_bin):
-    results = np.array(list(filter(lambda x: x is not None, results)))
-
-    np.set_printoptions(precision=3)
-
-    def _print_result(results):
-        res = {
-            "nutt": len(results),
-            "error": results[:, 0].mean(),
-            "std": results[:, 0].std(),
-            "dur_ref": int(results[:, 1].sum()),
-            "dur_syn": int(results[:, 2].sum()),
-        }
-        print(tabulate([res.values()], res.keys(), floatfmt=".4f"))
-
-    print(">>>> ALL")
-    _print_result(results)
-
-    if show_bin:
-        edges = [0, 200, 400, 600, 800, 1000, 2000, 4000]
-        for i in range(1, len(edges)):
-            mask = np.logical_and(results[:, 1] >= edges[i-1],
-                                  results[:, 1] < edges[i])
-            if not mask.any():
-                continue
-            bin_results = results[mask]
-            print(f">>>> ({edges[i-1]}, {edges[i]})")
-            _print_result(bin_results)
-
-
-def main(eval_f0, gpe, vde, ffe, show_bin):
-    samples = load_eval_spec(eval_f0)
-    if gpe or vde or ffe:
-        f0_samples = extract_f0(samples)
-
-    if gpe:
-        print("===== Evaluate Gross Pitch Error =====")
-        results = eval_gross_pitch_error(f0_samples)
-        print_results(results, show_bin)
-    if vde:
-        print("===== Evaluate Voicing Decision Error =====")
-        results = eval_voicing_decision_error(f0_samples)
-        print_results(results, show_bin)
-    if ffe:
-        print("===== Evaluate F0 Frame Error =====")
-        results = eval_f0_frame_error(f0_samples)
-        print_results(results, show_bin)
-
-
-if __name__ == "__main__":
-    import argparse
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument("eval_f0")
-    parser.add_argument("--gpe", action="store_true")
-    parser.add_argument("--vde", action="store_true")
-    parser.add_argument("--ffe", action="store_true")
-    parser.add_argument("--show-bin", action="store_true")
-    args = parser.parse_args()
-
-    main(args.eval_f0, args.gpe, args.vde, args.ffe, args.show_bin)

audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/evaluation/eval_sp.py

@@ -1,130 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-
-"""
-Signal processing-based evaluation using waveforms
-"""
-
-import csv
-import os.path as op
-
-import numpy as np
-import torch
-import torchaudio
-import tqdm
-from examples.speech_synthesis.utils import batch_mel_spectral_distortion
-from fairseq.tasks.text_to_speech import batch_mel_cepstral_distortion
-from tabulate import tabulate
-
-
-def load_eval_spec(path):
-    with open(path) as f:
-        reader = csv.DictReader(f, delimiter='\t')
-        samples = list(reader)
-    return samples
-
-
-def eval_distortion(samples, distortion_fn, device="cuda"):
-    nmiss = 0
-    results = []
-    for sample in tqdm.tqdm(samples):
-        if not op.isfile(sample["ref"]) or not op.isfile(sample["syn"]):
-            nmiss += 1
-            results.append(None)
-            continue
-        # assume single channel
-        yref, sr = torchaudio.load(sample["ref"])
-        ysyn, _sr = torchaudio.load(sample["syn"])
-        yref, ysyn = yref[0].to(device), ysyn[0].to(device)
-        assert sr == _sr, f"{sr} != {_sr}"
-
-        distortion, extra = distortion_fn([yref], [ysyn], sr, None)[0]
-        _, _, _, _, _, pathmap = extra
-        nins = torch.sum(pathmap.sum(dim=1) - 1)  # extra frames in syn
-        ndel = torch.sum(pathmap.sum(dim=0) - 1)  # missing frames from syn
-        results.append(
-            (distortion.item(),  # path distortion
-             pathmap.size(0),  # yref num frames
-             pathmap.size(1),  # ysyn num frames
-             pathmap.sum().item(),  # path length
-             nins.item(),  # insertion
-             ndel.item(),  # deletion
-             )
-        )
-    return results
-
-
-def eval_mel_cepstral_distortion(samples, device="cuda"):
-    return eval_distortion(samples, batch_mel_cepstral_distortion, device)
-
-
-def eval_mel_spectral_distortion(samples, device="cuda"):
-    return eval_distortion(samples, batch_mel_spectral_distortion, device)
-
-
-def print_results(results, show_bin):
-    results = np.array(list(filter(lambda x: x is not None, results)))
-
-    np.set_printoptions(precision=3)
-
-    def _print_result(results):
-        dist, dur_ref, dur_syn, dur_ali, nins, ndel = results.sum(axis=0)
-        res = {
-            "nutt": len(results),
-            "dist": dist,
-            "dur_ref": int(dur_ref),
-            "dur_syn": int(dur_syn),
-            "dur_ali": int(dur_ali),
-            "dist_per_ref_frm": dist/dur_ref,
-            "dist_per_syn_frm": dist/dur_syn,
-            "dist_per_ali_frm": dist/dur_ali,
-            "ins": nins/dur_ref,
-            "del": ndel/dur_ref,
-        }
-        print(tabulate(
-            [res.values()],
-            res.keys(),
-            floatfmt=".4f"
-        ))
-
-    print(">>>> ALL")
-    _print_result(results)
-
-    if show_bin:
-        edges = [0, 200, 400, 600, 800, 1000, 2000, 4000]
-        for i in range(1, len(edges)):
-            mask = np.logical_and(results[:, 1] >= edges[i-1],
-                                  results[:, 1] < edges[i])
-            if not mask.any():
-                continue
-            bin_results = results[mask]
-            print(f">>>> ({edges[i-1]}, {edges[i]})")
-            _print_result(bin_results)
-
-
-def main(eval_spec, mcd, msd, show_bin):
-    samples = load_eval_spec(eval_spec)
-    device = "cpu"
-    if mcd:
-        print("===== Evaluate Mean Cepstral Distortion =====")
-        results = eval_mel_cepstral_distortion(samples, device)
-        print_results(results, show_bin)
-    if msd:
-        print("===== Evaluate Mean Spectral Distortion =====")
-        results = eval_mel_spectral_distortion(samples, device)
-        print_results(results, show_bin)
-
-
-if __name__ == "__main__":
-    import argparse
-    parser = argparse.ArgumentParser()
-    parser.add_argument("eval_spec")
-    parser.add_argument("--mcd", action="store_true")
-    parser.add_argument("--msd", action="store_true")
-    parser.add_argument("--show-bin", action="store_true")
-    args = parser.parse_args()
-
-    main(args.eval_spec, args.mcd, args.msd, args.show_bin)

audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/evaluation/get_eval_manifest.py

@@ -1,64 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-
-import csv
-from pathlib import Path
-
-
-def main(args):
-    """
-    `uid syn ref text`
-    """
-    in_root = Path(args.generation_root).resolve()
-    ext = args.audio_format
-    with open(args.audio_manifest) as f, open(args.output_path, "w") as f_out:
-        reader = csv.DictReader(
-            f, delimiter="\t", quotechar=None, doublequote=False,
-            lineterminator="\n", quoting=csv.QUOTE_NONE
-        )
-        header = ["id", "syn", "ref", "text", "speaker"]
-        f_out.write("\t".join(header) + "\n")
-        for row in reader:
-            dir_name = f"{ext}_{args.sample_rate}hz_{args.vocoder}"
-            id_ = row["id"]
-            syn = (in_root / dir_name / f"{id_}.{ext}").as_posix()
-            ref = row["audio"]
-            if args.use_resynthesized_target:
-                ref = (in_root / f"{dir_name}_tgt" / f"{id_}.{ext}").as_posix()
-            if args.eval_target:
-                syn = row["audio"]
-            sample = [id_, syn, ref, row["tgt_text"], row["speaker"]]
-            f_out.write("\t".join(sample) + "\n")
-    print(f"wrote evaluation file to {args.output_path}")
-
-
-if __name__ == "__main__":
-    import argparse
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "--generation-root",  help="output directory for generate_waveform.py"
-    )
-    parser.add_argument(
-        "--audio-manifest",
-        help="used to determine the original utterance ID and text"
-    )
-    parser.add_argument(
-        "--output-path", help="path to output evaluation spec file"
-    )
-    parser.add_argument(
-        "--use-resynthesized-target", action="store_true",
-        help="use resynthesized reference instead of the original audio"
-    )
-    parser.add_argument(
-        "--eval-target", action="store_true",
-        help="evaluate reference instead of model prediction"
-    )
-    parser.add_argument("--vocoder", type=str, default="griffin_lim")
-    parser.add_argument("--sample-rate", type=int, default=22_050)
-    parser.add_argument("--audio-format", type=str, default="wav")
-    args = parser.parse_args()
-
-    main(args)

audio/safeear/safeear_repo/fairseq_ours/examples/speech_synthesis/generate_waveform.py

@@ -1,191 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-import ast
-import logging
-import sys
-from pathlib import Path
-
-import matplotlib.pyplot as plt
-import numpy as np
-import soundfile as sf
-import torch
-import torchaudio
-from fairseq import checkpoint_utils, options, tasks, utils
-from fairseq.data.audio.text_to_speech_dataset import TextToSpeechDataset
-from fairseq.logging import progress_bar
-from fairseq.tasks.text_to_speech import plot_tts_output
-
-logging.basicConfig()
-logging.root.setLevel(logging.INFO)
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
-
-
-def make_parser():
-    parser = options.get_speech_generation_parser()
-    parser.add_argument("--dump-features", action="store_true")
-    parser.add_argument("--dump-waveforms", action="store_true")
-    parser.add_argument("--dump-attentions", action="store_true")
-    parser.add_argument("--dump-eos-probs", action="store_true")
-    parser.add_argument("--dump-plots", action="store_true")
-    parser.add_argument("--dump-target", action="store_true")
-    parser.add_argument("--output-sample-rate", default=22050, type=int)
-    parser.add_argument("--teacher-forcing", action="store_true")
-    parser.add_argument(
-        "--audio-format", type=str, default="wav", choices=["wav", "flac"]
-    )
-    return parser
-
-
-def postprocess_results(
-        dataset: TextToSpeechDataset, sample, hypos, resample_fn, dump_target
-):
-    def to_np(x):
-        return None if x is None else x.detach().cpu().numpy()
-
-    sample_ids = [dataset.ids[i] for i in sample["id"].tolist()]
-    texts = sample["src_texts"] if "src_texts" in sample else [""] * len(hypos)
-    attns = [to_np(hypo["attn"]) for hypo in hypos]
-    eos_probs = [to_np(hypo.get("eos_prob", None)) for hypo in hypos]
-    feat_preds = [to_np(hypo["feature"]) for hypo in hypos]
-    wave_preds = [to_np(resample_fn(h["waveform"])) for h in hypos]
-    if dump_target:
-        feat_targs = [to_np(hypo["targ_feature"]) for hypo in hypos]
-        wave_targs = [to_np(resample_fn(h["targ_waveform"])) for h in hypos]
-    else:
-        feat_targs = [None for _ in hypos]
-        wave_targs = [None for _ in hypos]
-
-    return zip(sample_ids, texts, attns, eos_probs, feat_preds, wave_preds,
-               feat_targs, wave_targs)
-
-
-def dump_result(
-        is_na_model,
-        args,
-        vocoder,
-        sample_id,
-        text,
-        attn,
-        eos_prob,
-        feat_pred,
-        wave_pred,
-        feat_targ,
-        wave_targ,
-):
-    sample_rate = args.output_sample_rate
-    out_root = Path(args.results_path)
-    if args.dump_features:
-        feat_dir = out_root / "feat"
-        feat_dir.mkdir(exist_ok=True, parents=True)
-        np.save(feat_dir / f"{sample_id}.npy", feat_pred)
-        if args.dump_target:
-            feat_tgt_dir = out_root / "feat_tgt"
-            feat_tgt_dir.mkdir(exist_ok=True, parents=True)
-            np.save(feat_tgt_dir / f"{sample_id}.npy", feat_targ)
-    if args.dump_attentions:
-        attn_dir = out_root / "attn"
-        attn_dir.mkdir(exist_ok=True, parents=True)
-        np.save(attn_dir / f"{sample_id}.npy", attn.numpy())
-    if args.dump_eos_probs and not is_na_model:
-        eos_dir = out_root / "eos"
-        eos_dir.mkdir(exist_ok=True, parents=True)
-        np.save(eos_dir / f"{sample_id}.npy", eos_prob)
-
-    if args.dump_plots:
-        images = [feat_pred.T] if is_na_model else [feat_pred.T, attn]
-        names = ["output"] if is_na_model else ["output", "alignment"]
-        if feat_targ is not None:
-            images = [feat_targ.T] + images
-            names = [f"target (idx={sample_id})"] + names
-        if is_na_model:
-            plot_tts_output(images, names, attn, "alignment", suptitle=text)
-        else:
-            plot_tts_output(images, names, eos_prob, "eos prob", suptitle=text)
-        plot_dir = out_root / "plot"
-        plot_dir.mkdir(exist_ok=True, parents=True)
-        plt.savefig(plot_dir / f"{sample_id}.png")
-        plt.close()
-
-    if args.dump_waveforms:
-        ext = args.audio_format
-        if wave_pred is not None:
-            wav_dir = out_root / f"{ext}_{sample_rate}hz_{vocoder}"
-            wav_dir.mkdir(exist_ok=True, parents=True)
-            sf.write(wav_dir / f"{sample_id}.{ext}", wave_pred, sample_rate)
-        if args.dump_target and wave_targ is not None:
-            wav_tgt_dir = out_root / f"{ext}_{sample_rate}hz_{vocoder}_tgt"
-            wav_tgt_dir.mkdir(exist_ok=True, parents=True)
-            sf.write(wav_tgt_dir / f"{sample_id}.{ext}", wave_targ, sample_rate)
-
-
-def main(args):
-    assert(args.dump_features or args.dump_waveforms or args.dump_attentions
-           or args.dump_eos_probs or args.dump_plots)
-    if args.max_tokens is None and args.batch_size is None:
-        args.max_tokens = 8000
-    logger.info(args)
-
-    use_cuda = torch.cuda.is_available() and not args.cpu
-    task = tasks.setup_task(args)
-    models, saved_cfg, task = checkpoint_utils.load_model_ensemble_and_task(
-        [args.path],
-        task=task,
-        arg_overrides=ast.literal_eval(args.model_overrides),
-    )
-    model = models[0].cuda() if use_cuda else models[0]
-    # use the original n_frames_per_step
-    task.args.n_frames_per_step = saved_cfg.task.n_frames_per_step
-    task.load_dataset(args.gen_subset, task_cfg=saved_cfg.task)
-
-    data_cfg = task.data_cfg
-    sample_rate = data_cfg.config.get("features", {}).get("sample_rate", 22050)
-    resample_fn = {
-        False: lambda x: x,
-        True: lambda x: torchaudio.sox_effects.apply_effects_tensor(
-            x.detach().cpu().unsqueeze(0), sample_rate,
-            [['rate', str(args.output_sample_rate)]]
-        )[0].squeeze(0)
-    }.get(args.output_sample_rate != sample_rate)
-    if args.output_sample_rate != sample_rate:
-        logger.info(f"resampling to {args.output_sample_rate}Hz")
-
-    generator = task.build_generator([model], args)
-    itr = task.get_batch_iterator(
-        dataset=task.dataset(args.gen_subset),
-        max_tokens=args.max_tokens,
-        max_sentences=args.batch_size,
-        max_positions=(sys.maxsize, sys.maxsize),
-        ignore_invalid_inputs=args.skip_invalid_size_inputs_valid_test,
-        required_batch_size_multiple=args.required_batch_size_multiple,
-        num_shards=args.num_shards,
-        shard_id=args.shard_id,
-        num_workers=args.num_workers,
-        data_buffer_size=args.data_buffer_size,
-    ).next_epoch_itr(shuffle=False)
-
-    Path(args.results_path).mkdir(exist_ok=True, parents=True)
-    is_na_model = getattr(model, "NON_AUTOREGRESSIVE", False)
-    dataset = task.dataset(args.gen_subset)
-    vocoder = task.args.vocoder
-    with progress_bar.build_progress_bar(args, itr) as t:
-        for sample in t:
-            sample = utils.move_to_cuda(sample) if use_cuda else sample
-            hypos = generator.generate(model, sample, has_targ=args.dump_target)
-            for result in postprocess_results(
-                    dataset, sample, hypos, resample_fn, args.dump_target
-            ):
-                dump_result(is_na_model, args, vocoder, *result)
-
-
-def cli_main():
-    parser = make_parser()
-    args = options.parse_args_and_arch(parser)
-    main(args)
-
-
-if __name__ == "__main__":
-    cli_main()