chore: remove non-weight files (batch 3)

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/bart/model.py

@@ -1,393 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-"""
-BART: Denoising Sequence-to-Sequence Pre-training for
-Natural Language Generation, Translation, and Comprehension
-"""
-import logging
-from typing import Optional
-
-import torch
-import torch.nn as nn
-from fairseq import utils
-from fairseq.models import register_model, register_model_architecture
-from fairseq.models.transformer import TransformerModel
-from fairseq.modules.transformer_sentence_encoder import init_bert_params
-
-from .hub_interface import BARTHubInterface
-
-logger = logging.getLogger(__name__)
-
-
-@register_model("bart")
-class BARTModel(TransformerModel):
-    __jit_unused_properties__ = ["supported_targets"]
-
-    @classmethod
-    def hub_models(cls):
-        return {
-            "bart.base": "http://dl.fbaipublicfiles.com/fairseq/models/bart.base.tar.gz",
-            "bart.large": "http://dl.fbaipublicfiles.com/fairseq/models/bart.large.tar.gz",
-            "bart.large.mnli": "http://dl.fbaipublicfiles.com/fairseq/models/bart.large.mnli.tar.gz",
-            "bart.large.cnn": "http://dl.fbaipublicfiles.com/fairseq/models/bart.large.cnn.tar.gz",
-            "bart.large.xsum": "http://dl.fbaipublicfiles.com/fairseq/models/bart.large.xsum.tar.gz",
-        }
-
-    def __init__(self, args, encoder, decoder):
-        super().__init__(args, encoder, decoder)
-
-        # We follow BERT's random weight initialization
-        self.apply(init_bert_params)
-
-        self.classification_heads = nn.ModuleDict()
-        if hasattr(self.encoder, "dictionary"):
-            self.eos: int = self.encoder.dictionary.eos()
-
-    @staticmethod
-    def add_args(parser):
-        super(BARTModel, BARTModel).add_args(parser)
-        parser.add_argument(
-            "--pooler-dropout",
-            type=float,
-            metavar="D",
-            help="dropout probability in the masked_lm pooler layers",
-        )
-        parser.add_argument(
-            "--pooler-activation-fn",
-            choices=utils.get_available_activation_fns(),
-            help="activation function to use for pooler layer",
-        )
-        parser.add_argument(
-            "--spectral-norm-classification-head",
-            action="store_true",
-            help="Apply spectral normalization on the classification head",
-        )
-
-    @property
-    def supported_targets(self):
-        return {"self"}
-
-    def forward(
-        self,
-        src_tokens,
-        src_lengths,
-        prev_output_tokens,
-        features_only: bool = False,
-        classification_head_name: Optional[str] = None,
-        token_embeddings: Optional[torch.Tensor] = None,
-        return_all_hiddens: bool = True,
-        alignment_layer: Optional[int] = None,
-        alignment_heads: Optional[int] = None,
-    ):
-        if classification_head_name is not None:
-            features_only = True
-
-        encoder_out = self.encoder(
-            src_tokens,
-            src_lengths=src_lengths,
-            token_embeddings=token_embeddings,
-            return_all_hiddens=return_all_hiddens,
-        )
-        x, extra = self.decoder(
-            prev_output_tokens,
-            encoder_out=encoder_out,
-            features_only=features_only,
-            alignment_layer=alignment_layer,
-            alignment_heads=alignment_heads,
-            src_lengths=src_lengths,
-            return_all_hiddens=return_all_hiddens,
-        )
-        eos: int = self.eos
-        if classification_head_name is not None:
-            sentence_representation = x[src_tokens.eq(eos), :].view(
-                x.size(0), -1, x.size(-1)
-            )[:, -1, :]
-            for k, head in self.classification_heads.items():
-                # for torch script only supports iteration
-                if k == classification_head_name:
-                    x = head(sentence_representation)
-                    break
-        return x, extra
-
-    @classmethod
-    def from_pretrained(
-        cls,
-        model_name_or_path,
-        checkpoint_file="model.pt",
-        data_name_or_path=".",
-        bpe="gpt2",
-        sample_break_mode="eos",
-        **kwargs,
-    ):
-        from fairseq import hub_utils
-
-        x = hub_utils.from_pretrained(
-            model_name_or_path,
-            checkpoint_file,
-            data_name_or_path,
-            archive_map=cls.hub_models(),
-            bpe=bpe,
-            load_checkpoint_heads=True,
-            sample_break_mode=sample_break_mode,
-            **kwargs,
-        )
-        return BARTHubInterface(x["args"], x["task"], x["models"][0])
-
-    def register_classification_head(
-        self, name, num_classes=None, inner_dim=None, **kwargs
-    ):
-        """Register a classification head."""
-        logger.info("Registering classification head: {0}".format(name))
-        if name in self.classification_heads:
-            prev_num_classes = self.classification_heads[name].out_proj.out_features
-            prev_inner_dim = self.classification_heads[name].dense.out_features
-            if num_classes != prev_num_classes or inner_dim != prev_inner_dim:
-                logger.warning(
-                    're-registering head "{}" with num_classes {} (prev: {}) '
-                    "and inner_dim {} (prev: {})".format(
-                        name, num_classes, prev_num_classes, inner_dim, prev_inner_dim
-                    )
-                )
-        self.classification_heads[name] = BARTClassificationHead(
-            input_dim=self.args.encoder_embed_dim,
-            inner_dim=inner_dim or self.args.encoder_embed_dim,
-            num_classes=num_classes,
-            activation_fn=self.args.pooler_activation_fn,
-            pooler_dropout=self.args.pooler_dropout,
-            do_spectral_norm=getattr(
-                self.args, "spectral_norm_classification_head", False
-            ),
-        )
-
-    def upgrade_state_dict_named(self, state_dict, name):
-        super().upgrade_state_dict_named(state_dict, name)
-
-        prefix = name + "." if name != "" else ""
-        current_head_names = (
-            []
-            if not hasattr(self, "classification_heads")
-            else self.classification_heads.keys()
-        )
-
-        # Handle new classification heads present in the state dict.
-        keys_to_delete = []
-        for k in state_dict.keys():
-            if not k.startswith(prefix + "classification_heads."):
-                continue
-
-            head_name = k[len(prefix + "classification_heads.") :].split(".")[0]
-            num_classes = state_dict[
-                prefix + "classification_heads." + head_name + ".out_proj.weight"
-            ].size(0)
-            inner_dim = state_dict[
-                prefix + "classification_heads." + head_name + ".dense.weight"
-            ].size(0)
-
-            if getattr(self.args, "load_checkpoint_heads", False):
-                if head_name not in current_head_names:
-                    self.register_classification_head(head_name, num_classes, inner_dim)
-            else:
-                if head_name not in current_head_names:
-                    logger.warning(
-                        "deleting classification head ({}) from checkpoint "
-                        "not present in current model: {}".format(head_name, k)
-                    )
-                    keys_to_delete.append(k)
-                elif (
-                    num_classes
-                    != self.classification_heads[head_name].out_proj.out_features
-                    or inner_dim
-                    != self.classification_heads[head_name].dense.out_features
-                ):
-                    logger.warning(
-                        "deleting classification head ({}) from checkpoint "
-                        "with different dimensions than current model: {}".format(
-                            head_name, k
-                        )
-                    )
-                    keys_to_delete.append(k)
-        for k in keys_to_delete:
-            del state_dict[k]
-
-        def truncate_emb(key):
-            if key in state_dict:
-                state_dict[key] = state_dict[key][:-1, :]
-
-        # When finetuning on translation task, remove last row of
-        # embedding matrix that corresponds to mask_idx token.
-        loaded_dict_size = state_dict["encoder.embed_tokens.weight"].size(0)
-        if (
-            loaded_dict_size == len(self.encoder.dictionary) + 1
-            and "<mask>" not in self.encoder.dictionary
-        ):
-            truncate_emb("encoder.embed_tokens.weight")
-            truncate_emb("decoder.embed_tokens.weight")
-            truncate_emb("encoder.output_projection.weight")
-            truncate_emb("decoder.output_projection.weight")
-
-        # When continued pretraining on new set of languages for mbart,
-        # add extra lang embeddings at the end of embed_tokens.
-        # Note: newly added languages are assumed to have been added at the end.
-        if self.args.task == "multilingual_denoising" and loaded_dict_size < len(
-            self.encoder.dictionary
-        ):
-            logger.info(
-                "Adding extra language embeddings not found in pretrained model for "
-                "continued pretraining of MBART on new set of languages."
-            )
-            loaded_mask_token_embedding = state_dict["encoder.embed_tokens.weight"][
-                -1, :
-            ]
-
-            num_langids_to_add = len(self.encoder.dictionary) - loaded_dict_size
-            embed_dim = state_dict["encoder.embed_tokens.weight"].size(1)
-
-            new_lang_embed_to_add = torch.zeros(num_langids_to_add, embed_dim)
-            nn.init.normal_(new_lang_embed_to_add, mean=0, std=embed_dim**-0.5)
-            new_lang_embed_to_add = new_lang_embed_to_add.to(
-                dtype=state_dict["encoder.embed_tokens.weight"].dtype,
-            )
-
-            state_dict["encoder.embed_tokens.weight"] = torch.cat(
-                [
-                    state_dict["encoder.embed_tokens.weight"][
-                        : loaded_dict_size - 1, :
-                    ],
-                    new_lang_embed_to_add,
-                    loaded_mask_token_embedding.unsqueeze(0),
-                ]
-            )
-            state_dict["decoder.embed_tokens.weight"] = torch.cat(
-                [
-                    state_dict["decoder.embed_tokens.weight"][
-                        : loaded_dict_size - 1, :
-                    ],
-                    new_lang_embed_to_add,
-                    loaded_mask_token_embedding.unsqueeze(0),
-                ]
-            )
-
-        # Copy any newly-added classification heads into the state dict
-        # with their current weights.
-        if hasattr(self, "classification_heads"):
-            cur_state = self.classification_heads.state_dict()
-            for k, v in cur_state.items():
-                if prefix + "classification_heads." + k not in state_dict:
-                    logger.info("Overwriting " + prefix + "classification_heads." + k)
-                    state_dict[prefix + "classification_heads." + k] = v
-
-    def set_beam_size(self, beam):
-        """Set beam size for efficient beamable enc-dec attention."""
-        beamable = False
-        for layer in self.decoder.layers:
-            if layer.encoder_attn is not None:
-                if hasattr(layer.encoder_attn, "set_beam_size"):
-                    layer.encoder_attn.set_beam_size(beam)
-                    beamable = True
-        if beamable:
-            self.encoder.reorder_encoder_out = self.encoder._reorder_encoder_out
-
-
-class BARTClassificationHead(nn.Module):
-    """Head for sentence-level classification tasks."""
-
-    def __init__(
-        self,
-        input_dim,
-        inner_dim,
-        num_classes,
-        activation_fn,
-        pooler_dropout,
-        do_spectral_norm=False,
-    ):
-        super().__init__()
-        self.dense = nn.Linear(input_dim, inner_dim)
-        self.activation_fn = utils.get_activation_fn(activation_fn)
-        self.dropout = nn.Dropout(p=pooler_dropout)
-        self.out_proj = nn.Linear(inner_dim, num_classes)
-
-        if do_spectral_norm:
-            self.out_proj = torch.nn.utils.spectral_norm(self.out_proj)
-
-    def forward(self, features, **kwargs):
-        x = features
-        x = self.dropout(x)
-        x = self.dense(x)
-        x = self.activation_fn(x)
-        x = self.dropout(x)
-        x = self.out_proj(x)
-        return x
-
-
-@register_model_architecture("bart", "bart_large")
-def bart_large_architecture(args):
-    args.encoder_embed_path = getattr(args, "encoder_embed_path", None)
-    args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 1024)
-    args.encoder_ffn_embed_dim = getattr(args, "encoder_ffn_embed_dim", 4 * 1024)
-    args.encoder_layers = getattr(args, "encoder_layers", 12)
-    args.encoder_attention_heads = getattr(args, "encoder_attention_heads", 16)
-    args.encoder_normalize_before = getattr(args, "encoder_normalize_before", False)
-    args.encoder_learned_pos = getattr(args, "encoder_learned_pos", True)
-    args.decoder_embed_path = getattr(args, "decoder_embed_path", None)
-    args.decoder_embed_dim = getattr(args, "decoder_embed_dim", args.encoder_embed_dim)
-    args.decoder_ffn_embed_dim = getattr(
-        args, "decoder_ffn_embed_dim", args.encoder_ffn_embed_dim
-    )
-    args.decoder_layers = getattr(args, "decoder_layers", 12)
-    args.decoder_attention_heads = getattr(args, "decoder_attention_heads", 16)
-    args.decoder_normalize_before = getattr(args, "decoder_normalize_before", False)
-    args.decoder_learned_pos = getattr(args, "decoder_learned_pos", True)
-    args.attention_dropout = getattr(args, "attention_dropout", 0.0)
-    args.relu_dropout = getattr(args, "relu_dropout", 0.0)
-    args.dropout = getattr(args, "dropout", 0.1)
-    args.max_target_positions = getattr(args, "max_target_positions", 1024)
-    args.max_source_positions = getattr(args, "max_source_positions", 1024)
-    args.adaptive_softmax_cutoff = getattr(args, "adaptive_softmax_cutoff", None)
-    args.adaptive_softmax_dropout = getattr(args, "adaptive_softmax_dropout", 0)
-    args.share_decoder_input_output_embed = getattr(
-        args, "share_decoder_input_output_embed", True
-    )
-    args.share_all_embeddings = getattr(args, "share_all_embeddings", True)
-
-    args.decoder_output_dim = getattr(
-        args, "decoder_output_dim", args.decoder_embed_dim
-    )
-    args.decoder_input_dim = getattr(args, "decoder_input_dim", args.decoder_embed_dim)
-
-    args.no_scale_embedding = getattr(args, "no_scale_embedding", True)
-    args.layernorm_embedding = getattr(args, "layernorm_embedding", True)
-
-    args.activation_fn = getattr(args, "activation_fn", "gelu")
-    args.pooler_activation_fn = getattr(args, "pooler_activation_fn", "tanh")
-    args.pooler_dropout = getattr(args, "pooler_dropout", 0.0)
-
-
-@register_model_architecture("bart", "bart_base")
-def bart_base_architecture(args):
-    args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 768)
-    args.encoder_ffn_embed_dim = getattr(args, "encoder_ffn_embed_dim", 4 * 768)
-    args.encoder_layers = getattr(args, "encoder_layers", 6)
-    args.encoder_attention_heads = getattr(args, "encoder_attention_heads", 12)
-    args.decoder_layers = getattr(args, "decoder_layers", 6)
-    args.decoder_attention_heads = getattr(args, "decoder_attention_heads", 12)
-    bart_large_architecture(args)
-
-
-@register_model_architecture("bart", "mbart_large")
-def mbart_large_architecture(args):
-    args.no_scale_embedding = getattr(args, "no_scale_embedding", False)
-    bart_large_architecture(args)
-
-
-@register_model_architecture("bart", "mbart_base")
-def mbart_base_architecture(args):
-    args.no_scale_embedding = getattr(args, "no_scale_embedding", False)
-    bart_base_architecture(args)
-
-
-@register_model_architecture("bart", "mbart_base_wmt20")
-def mbart_base_wmt20_architecture(args):
-    args.layernorm_embedding = getattr(args, "layernorm_embedding", False)
-    mbart_base_architecture(args)

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/composite_encoder.py

@@ -1,57 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-from .fairseq_encoder import FairseqEncoder
-
-
-class CompositeEncoder(FairseqEncoder):
-    """
-    A wrapper around a dictionary of :class:`FairseqEncoder` objects.
-
-    We run forward on each encoder and return a dictionary of outputs. The first
-    encoder's dictionary is used for initialization.
-
-    Args:
-        encoders (dict): a dictionary of :class:`FairseqEncoder` objects.
-    """
-
-    def __init__(self, encoders):
-        super().__init__(next(iter(encoders.values())).dictionary)
-        self.encoders = encoders
-        for key in self.encoders:
-            self.add_module(key, self.encoders[key])
-
-    def forward(self, src_tokens, src_lengths):
-        """
-        Args:
-            src_tokens (LongTensor): tokens in the source language of shape
-                `(batch, src_len)`
-            src_lengths (LongTensor): lengths of each source sentence of shape
-                `(batch)`
-
-        Returns:
-            dict:
-                the outputs from each Encoder
-        """
-        encoder_out = {}
-        for key in self.encoders:
-            encoder_out[key] = self.encoders[key](src_tokens, src_lengths)
-        return encoder_out
-
-    def reorder_encoder_out(self, encoder_out, new_order):
-        """Reorder encoder output according to new_order."""
-        for key in self.encoders:
-            encoder_out[key] = self.encoders[key].reorder_encoder_out(
-                encoder_out[key], new_order
-            )
-        return encoder_out
-
-    def max_positions(self):
-        return min(self.encoders[key].max_positions() for key in self.encoders)
-
-    def upgrade_state_dict(self, state_dict):
-        for key in self.encoders:
-            self.encoders[key].upgrade_state_dict(state_dict)
-        return state_dict

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/distributed_fairseq_model.py

@@ -1,146 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-import logging
-import os
-import signal
-import threading
-
-import torch
-import torch.nn as nn
-from fairseq.distributed import (
-    DistributedTimeoutWrapper,
-    LegacyDistributedDataParallel,
-    ModuleProxyWrapper,
-    TPUDistributedDataParallel,
-)
-from torch.nn.parallel import DistributedDataParallel
-
-logger = logging.getLogger(__name__)
-
-
-_SLOWMO_DDP_DISABLED = False
-try:
-    from fairscale.experimental.nn.data_parallel import (
-        SlowMoBaseAlgorithm,
-        SlowMoDistributedDataParallel,
-    )
-except ImportError:
-    _SLOWMO_DDP_DISABLED = True
-
-
-def DistributedFairseqModel(args, model, process_group, device):
-    """
-    Wrap a *model* to support distributed data parallel training.
-
-    This is similar to the built-in DistributedDataParallel, but allows
-    additional configuration of the DistributedDataParallel class to
-    use, and also provides easier access to the wrapped model by
-    forwarding requests for missing attributes to the wrapped model.
-
-    Args:
-        args (argparse.Namespace): fairseq args
-        model (BaseFairseqModel): model to wrap
-        process_group: the c10d process group to be used for distributed data
-            parallel all-reduction.
-        device: device to move model to
-    """
-    assert isinstance(model, nn.Module)
-    if args.tpu:
-        wrapped_model = TPUDistributedDataParallel(
-            module=model.to(device),
-            process_group=process_group,
-        )
-        # forward missing getattr and state_dict/load_state_dict to orig model
-        wrapped_model = ModuleProxyWrapper(wrapped_model)
-    elif args.ddp_backend in {"c10d", "pytorch_ddp"}:
-        wrapped_model = DistributedDataParallel(
-            module=model.to(device),
-            device_ids=[args.device_id],
-            output_device=args.device_id,
-            broadcast_buffers=args.broadcast_buffers,
-            bucket_cap_mb=args.bucket_cap_mb,
-            process_group=process_group,
-            find_unused_parameters=args.find_unused_parameters,
-            gradient_as_bucket_view=args.gradient_as_bucket_view,
-        )
-        if args.ddp_comm_hook == "fp16":
-            logger.info("enable fp16 communication hook in DDP")
-            try:
-                from torch.distributed.algorithms.ddp_comm_hooks import (
-                    DDPCommHookType,
-                    register_ddp_comm_hook,
-                )
-            except:
-                logger.error(
-                    "Could not import from torch.distributed.algorithms.ddp_comm_hooks; you may need to update your pytorch version"
-                )
-                raise
-
-            register_ddp_comm_hook(DDPCommHookType.FP16_COMPRESS, wrapped_model)
-        # forward missing getattr and state_dict/load_state_dict to orig model
-        wrapped_model = ModuleProxyWrapper(wrapped_model)
-    elif args.ddp_backend in {"no_c10d", "legacy_ddp"}:
-        wrapped_model = LegacyDistributedDataParallel(
-            module=model.to(device),
-            buffer_size=2**28,
-            process_group=process_group,
-        )
-        # forward missing getattr and state_dict/load_state_dict to orig model
-        wrapped_model = ModuleProxyWrapper(wrapped_model)
-    elif args.ddp_backend == "slowmo":
-        if _SLOWMO_DDP_DISABLED:
-            raise ImportError(
-                "Cannot find SlowMoDistributedDataParallel. "
-                "Please install fairscale with: pip install fairscale"
-            )
-
-        # The values of slowmo_momentum below were obtained by tuning on the
-        # En-De 16 dataset by training the transformer_wmt_en_de_large model
-        if args.slowmo_momentum is None:
-            if args.distributed_world_size <= 16:
-                args.slowmo_momentum = 0.0
-            elif args.distributed_world_size <= 32:
-                args.slowmo_momentum = 0.2
-            elif args.distributed_world_size <= 64:
-                args.slowmo_momentum = 0.5
-            else:
-                args.slowmo_momentum = 0.6
-        slowmo_base_algorithm = SlowMoBaseAlgorithm[args.slowmo_base_algorithm.upper()]
-
-        wrapped_model = SlowMoDistributedDataParallel(
-            module=model.to(device),
-            broadcast_buffers=args.broadcast_buffers,
-            nprocs_per_node=args.nprocs_per_node,
-            slowmo_momentum=args.slowmo_momentum,
-            slowmo_base_algorithm=slowmo_base_algorithm,
-            localsgd_frequency=args.localsgd_frequency,
-        )
-        # forward missing getattr and state_dict/load_state_dict to orig model
-        wrapped_model = ModuleProxyWrapper(wrapped_model)
-    elif args.ddp_backend == "fully_sharded":
-        try:
-            from fairscale.nn.data_parallel import FullyShardedDataParallel as FSDP
-        except ImportError:
-            raise ImportError(
-                "Cannot find FullyShardedDataParallel. "
-                "Please install fairscale with: pip install fairscale"
-            )
-        assert isinstance(model, FSDP), "expected model to already be wrapped in FSDP"
-        wrapped_model = model
-        if args.memory_efficient_fp16:
-            wrapped_model = wrapped_model.half()
-        if not args.cpu_offload:
-            wrapped_model = wrapped_model.to(device=device)
-    else:
-        raise ValueError("Unknown --ddp-backend: " + args.ddp_backend)
-
-    # kill hung distributed jobs after a timeout
-    if getattr(args, "heartbeat_timeout", -1) > 0:
-        wrapped_model = DistributedTimeoutWrapper(
-            wrapped_model, timeout=getattr(args, "heartbeat_timeout", -1)
-        )
-
-    return wrapped_model

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/ema/__init__.py

@@ -1,20 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-import importlib
-import os
-
-from .ema import EMA
-
-
-def build_ema(model, cfg, device):
-    return EMA(model, cfg, device)
-
-
-# automatically import any Python files in the models/ema/ directory
-for file in sorted(os.listdir(os.path.dirname(__file__))):
-    if file.endswith(".py") and not file.startswith("_"):
-        file_name = file[: file.find(".py")]
-        importlib.import_module("fairseq.models.ema." + file_name)

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/ema/ema.py

@@ -1,208 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-This module has the EMA class used to store a copy of the exponentially decayed
-model params.
-
-Typical usage of EMA class involves initializing an object using an existing
-model (random or from a seed model) and setting the config like ema_decay,
-ema_start_update which determine how the EMA model is updated. After every
-update of the model i.e. at the end of the train_step, the EMA should be updated
-by passing the new model to the EMA.step function. The EMA model state dict
-can be stored in the extra state under the key of "ema" and dumped
-into a checkpoint and loaded. The EMA object can be passed to tasks
-by setting task.uses_ema property.
-EMA is a smoothed/ensemble model which might have better performance
-when used for inference or further fine-tuning. EMA class has a
-reverse function to load the EMA params into a model and use it
-like a regular model.
-
-This implementation is used for trainer-level ema tracking. For EMA tracking
-inside the model, please use fairseq/modules/ema_module.py instead.
-"""
-
-import copy
-import logging
-
-import torch
-from fairseq import checkpoint_utils
-
-
-class EMA(object):
-    """Exponential Moving Average of Fairseq Models
-    EMA keeps a copy of the exponentially decayed model params.
-    The set of params should include both gradient-descent and
-    non-gradient descent params, such as batch mean/var and buffers.
-    This is a modified implementation of
-    the open source code in https://github.com/zhawe01/fairseq-gec.git,
-    and internal source code in
-    fbcode/mobile-vision/projects/classification_pytorch/lib/utils/model_ema.py.
-
-    Similar to TF EMA.
-    https://www.tensorflow.org/api_docs/python/tf/train/ExponentialMovingAverage.
-    EMA provides a averaged and smoothed set of model weights, and has been shown to
-    improve vision models. EMA class does all necessary functions to update, reload,
-    or init EMA methods.
-
-    EMA object is initialized from an arbitrary model. By default, it is stored in
-    the same device (unless device specified at initialization) and with the
-    same precision as the model (unless ema_fp32 is True). ema_fp32 is recommended.
-    This stores the EMA parameters in fp32 only for the EMA update step, and
-    is used at the default precision otherwise.
-    EMA is usually enabled using EMAConfig with store_ema=True. Some important
-    parameters to configure EMA are
-    1) ema_decay - The decay of EMA
-    2) ema_update_freq - EMA is updated every this many model updates.
-    3) ema_start_update - Start EMA update after this many model updates [default 0]
-
-    Key methods:
-    1) step - One update of EMA using new model
-    2) restore - Update EMA from a state dict
-    3) reverse - Load EMA into a model
-    4) get_decay, _set_decay - Used to get or set the decay.  Note _set_decay is
-    called from step.
-    5) build_fp32_params - Used to initialize or update the fp32 copy of EMA params.
-    Note this is enabled only when ema_fp32=True
-    """
-
-    def __init__(self, model, config, device=None, skip_keys=None):
-        """
-        @param model model to initialize the EMA with
-        @param config EMAConfig object with configuration like
-        ema_decay, ema_update_freq, ema_fp32
-        @param device If provided, copy EMA to this device (e.g. gpu).
-        Otherwise EMA is in the same device as the model.
-        """
-
-        self.decay = config.ema_decay
-        self.model = copy.deepcopy(model)
-        self.model.requires_grad_(False)
-        self.config = config
-        self.skip_keys = skip_keys or set()
-        self.fp32_params = {}
-
-        if self.config.ema_seed_model is not None:
-            state = checkpoint_utils.load_ema_from_checkpoint(
-                self.config.ema_seed_model
-            )
-            self.model.load_state_dict(state["model"], strict=True)
-
-        if device is not None:
-            logging.info(f"Copying EMA model to device {device}")
-            self.model = self.model.to(device=device)
-
-        if self.config.ema_fp32:
-            self.build_fp32_params()
-
-        self.update_freq_counter = 0
-
-    def get_model(self):
-        return self.model
-
-    def build_fp32_params(self, state_dict=None):
-        """
-        Store a copy of the EMA params in fp32.
-        If state dict is passed, the EMA params is copied from
-        the provided state dict. Otherwise, it is copied from the
-        current EMA model parameters.
-        """
-        if not self.config.ema_fp32:
-            raise RuntimeError(
-                "build_fp32_params should not be called if ema_fp32=False. "
-                "Use ema_fp32=True if this is really intended."
-            )
-
-        if state_dict is None:
-            state_dict = self.model.state_dict()
-
-        def _to_float(t):
-            return t.float() if torch.is_floating_point(t) else t
-
-        for param_key in state_dict:
-            if param_key in self.fp32_params:
-                self.fp32_params[param_key].copy_(state_dict[param_key])
-            else:
-                self.fp32_params[param_key] = _to_float(state_dict[param_key])
-
-    def restore(self, state_dict, build_fp32_params=False):
-        """Load data from a model spec into EMA model"""
-        self.model.load_state_dict(state_dict, strict=False)
-        if build_fp32_params:
-            self.build_fp32_params(state_dict)
-
-    def _set_decay(self, decay):
-        self.decay = decay
-
-    def get_decay(self):
-        return self.decay
-
-    def _step_internal(self, new_model, updates=None):
-        """One update of the EMA model based on new model weights"""
-        decay = self.decay
-
-        ema_state_dict = {}
-        ema_params = (
-            self.fp32_params if self.config.ema_fp32 else self.model.state_dict()
-        )
-        for key, param in new_model.state_dict().items():
-            if isinstance(param, dict):
-                continue
-            try:
-                ema_param = ema_params[key]
-            except KeyError:
-                ema_param = (
-                    param.float().clone() if param.ndim == 1 else copy.deepcopy(param)
-                )
-
-            if param.shape != ema_param.shape:
-                raise ValueError(
-                    "incompatible tensor shapes between model param and ema param"
-                    + "{} vs. {}".format(param.shape, ema_param.shape)
-                )
-
-            if "version" in key:
-                # Do not decay a model.version pytorch param
-                continue
-
-            if key in self.skip_keys:
-                ema_param = param.to(dtype=ema_param.dtype).clone()
-            else:
-                ema_param.mul_(decay)
-                ema_param.add_(param.to(dtype=ema_param.dtype), alpha=1 - decay)
-            ema_state_dict[key] = ema_param
-        self.restore(ema_state_dict, build_fp32_params=False)
-
-    def step(self, new_model, updates=None):
-        """
-        One update of EMA which is done every self.config.ema_update_freq
-        updates of the model.
-
-        @param updates The current number of model updates done.
-        Decay is set of 0 if model updates < ema_start_update, which means
-        the model will be simply copied over to the EMA.
-        When model updates >= ema_start_updates, then EMA is updated with
-        a decay of self.config.ema_decay.
-        """
-        if updates is not None:
-            self._set_decay(
-                0 if updates < self.config.ema_start_update else self.config.ema_decay
-            )
-        if self.config.ema_update_freq > 1:
-            self.update_freq_counter += 1
-            if self.update_freq_counter >= self.config.ema_update_freq:
-                self._step_internal(new_model, updates)
-                self.update_freq_counter = 0
-        else:
-            self._step_internal(new_model, updates)
-
-    def reverse(self, model):
-        """
-        Load the model parameters from EMA model.
-        Useful for inference or fine-tuning from the EMA model.
-        """
-        d = self.model.state_dict()
-        if "_ema" in d:
-            del d["_ema"]
-
-        model.load_state_dict(d, strict=False)
-        return model

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/fairseq_decoder.py

@@ -1,104 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-from typing import Dict, List, Optional, Tuple
-
-import torch.nn as nn
-from fairseq import utils
-from torch import Tensor
-
-
-class FairseqDecoder(nn.Module):
-    """Base class for decoders."""
-
-    def __init__(self, dictionary):
-        super().__init__()
-        self.dictionary = dictionary
-        self.onnx_trace = False
-        self.adaptive_softmax = None
-
-    def forward(self, prev_output_tokens, encoder_out=None, **kwargs):
-        """
-        Args:
-            prev_output_tokens (LongTensor): shifted output tokens of shape
-                `(batch, tgt_len)`, for teacher forcing
-            encoder_out (dict, optional): output from the encoder, used for
-                encoder-side attention
-
-        Returns:
-            tuple:
-                - the decoder's output of shape `(batch, tgt_len, vocab)`
-                - a dictionary with any model-specific outputs
-        """
-        x, extra = self.extract_features(
-            prev_output_tokens, encoder_out=encoder_out, **kwargs
-        )
-        x = self.output_layer(x)
-        return x, extra
-
-    def extract_features(self, prev_output_tokens, encoder_out=None, **kwargs):
-        """
-        Returns:
-            tuple:
-                - the decoder's features of shape `(batch, tgt_len, embed_dim)`
-                - a dictionary with any model-specific outputs
-        """
-        raise NotImplementedError
-
-    def output_layer(self, features, **kwargs):
-        """
-        Project features to the default output size, e.g., vocabulary size.
-
-        Args:
-            features (Tensor): features returned by *extract_features*.
-        """
-        raise NotImplementedError
-
-    def get_normalized_probs(
-        self,
-        net_output: Tuple[Tensor, Optional[Dict[str, List[Optional[Tensor]]]]],
-        log_probs: bool,
-        sample: Optional[Dict[str, Tensor]] = None,
-    ):
-        """Get normalized probabilities (or log probs) from a net's output."""
-        return self.get_normalized_probs_scriptable(net_output, log_probs, sample)
-
-    # TorchScript doesn't support super() method so that the scriptable Subclass
-    # can't access the base class model in Torchscript.
-    # Current workaround is to add a helper function with different name and
-    # call the helper function from scriptable Subclass.
-    def get_normalized_probs_scriptable(
-        self,
-        net_output: Tuple[Tensor, Optional[Dict[str, List[Optional[Tensor]]]]],
-        log_probs: bool,
-        sample: Optional[Dict[str, Tensor]] = None,
-    ):
-        """Get normalized probabilities (or log probs) from a net's output."""
-
-        if hasattr(self, "adaptive_softmax") and self.adaptive_softmax is not None:
-            if sample is not None:
-                assert "target" in sample
-                target = sample["target"]
-            else:
-                target = None
-            out = self.adaptive_softmax.get_log_prob(net_output[0], target=target)
-            return out.exp_() if not log_probs else out
-
-        logits = net_output[0]
-        if log_probs:
-            return utils.log_softmax(logits, dim=-1, onnx_trace=self.onnx_trace)
-        else:
-            return utils.softmax(logits, dim=-1, onnx_trace=self.onnx_trace)
-
-    def max_positions(self):
-        """Maximum input length supported by the decoder."""
-        return 1e6  # an arbitrary large number
-
-    def upgrade_state_dict_named(self, state_dict, name):
-        """Upgrade old state dicts to work with newer code."""
-        return state_dict
-
-    def prepare_for_onnx_export_(self):
-        self.onnx_trace = True

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/fairseq_encoder.py

@@ -1,91 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-from typing import Dict, List, NamedTuple, Optional
-
-import torch
-import torch.nn as nn
-from torch import Tensor
-
-EncoderOut = NamedTuple(
-    "EncoderOut",
-    [
-        ("encoder_out", Tensor),  # T x B x C
-        ("encoder_padding_mask", Optional[Tensor]),  # B x T
-        ("encoder_embedding", Optional[Tensor]),  # B x T x C
-        ("encoder_states", Optional[List[Tensor]]),  # List[T x B x C]
-        ("src_tokens", Optional[Tensor]),  # B x T
-        ("src_lengths", Optional[Tensor]),  # B x 1
-    ],
-)
-
-
-class FairseqEncoder(nn.Module):
-    """Base class for encoders."""
-
-    def __init__(self, dictionary):
-        super().__init__()
-        self.dictionary = dictionary
-
-    def forward(self, src_tokens, src_lengths=None, **kwargs):
-        """
-        Args:
-            src_tokens (LongTensor): tokens in the source language of shape
-                `(batch, src_len)`
-            src_lengths (LongTensor): lengths of each source sentence of shape
-                `(batch)`
-        """
-        raise NotImplementedError
-
-    def forward_torchscript(self, net_input: Dict[str, Tensor]):
-        """A TorchScript-compatible version of forward.
-
-        Encoders which use additional arguments may want to override
-        this method for TorchScript compatibility.
-        """
-        if torch.jit.is_scripting():
-            return self.forward(
-                src_tokens=net_input["src_tokens"],
-                src_lengths=net_input["src_lengths"],
-            )
-        else:
-            return self.forward_non_torchscript(net_input)
-
-    @torch.jit.unused
-    def forward_non_torchscript(self, net_input: Dict[str, Tensor]):
-        encoder_input = {
-            k: v for k, v in net_input.items() if k != "prev_output_tokens"
-        }
-        return self.forward(**encoder_input)
-
-    def reorder_encoder_out(self, encoder_out, new_order):
-        """
-        Reorder encoder output according to `new_order`.
-
-        Args:
-            encoder_out: output from the ``forward()`` method
-            new_order (LongTensor): desired order
-
-        Returns:
-            `encoder_out` rearranged according to `new_order`
-        """
-        raise NotImplementedError
-
-    def max_positions(self):
-        """Maximum input length supported by the encoder."""
-        return 1e6  # an arbitrary large number
-
-    def upgrade_state_dict_named(self, state_dict, name):
-        """Upgrade old state dicts to work with newer code."""
-        return state_dict
-
-    def set_num_updates(self, num_updates):
-        """State from trainer to pass along to model at every update."""
-
-        def _apply(m):
-            if hasattr(m, "set_num_updates") and m != self:
-                m.set_num_updates(num_updates)
-
-        self.apply(_apply)

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/fairseq_incremental_decoder.py

@@ -1,117 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-import logging
-from typing import Dict, Optional
-
-from fairseq.incremental_decoding_utils import with_incremental_state
-from fairseq.models import FairseqDecoder
-from torch import Tensor
-
-logger = logging.getLogger(__name__)
-
-
-@with_incremental_state
-class FairseqIncrementalDecoder(FairseqDecoder):
-    """Base class for incremental decoders.
-
-    Incremental decoding is a special mode at inference time where the Model
-    only receives a single timestep of input corresponding to the previous
-    output token (for teacher forcing) and must produce the next output
-    *incrementally*. Thus the model must cache any long-term state that is
-    needed about the sequence, e.g., hidden states, convolutional states, etc.
-
-    Compared to the standard :class:`FairseqDecoder` interface, the incremental
-    decoder interface allows :func:`forward` functions to take an extra keyword
-    argument (*incremental_state*) that can be used to cache state across
-    time-steps.
-
-    The :class:`FairseqIncrementalDecoder` interface also defines the
-    :func:`reorder_incremental_state` method, which is used during beam search
-    to select and reorder the incremental state based on the selection of beams.
-
-    To learn more about how incremental decoding works, refer to `this blog
-    <http://www.telesens.co/2019/04/21/understanding-incremental-decoding-in-fairseq/>`_.
-    """
-
-    def __init__(self, dictionary):
-        super().__init__(dictionary)
-
-    def forward(
-        self, prev_output_tokens, encoder_out=None, incremental_state=None, **kwargs
-    ):
-        """
-        Args:
-            prev_output_tokens (LongTensor): shifted output tokens of shape
-                `(batch, tgt_len)`, for teacher forcing
-            encoder_out (dict, optional): output from the encoder, used for
-                encoder-side attention
-            incremental_state (dict, optional): dictionary used for storing
-                state during :ref:`Incremental decoding`
-
-        Returns:
-            tuple:
-                - the decoder's output of shape `(batch, tgt_len, vocab)`
-                - a dictionary with any model-specific outputs
-        """
-        raise NotImplementedError
-
-    def extract_features(
-        self, prev_output_tokens, encoder_out=None, incremental_state=None, **kwargs
-    ):
-        """
-        Returns:
-            tuple:
-                - the decoder's features of shape `(batch, tgt_len, embed_dim)`
-                - a dictionary with any model-specific outputs
-        """
-        raise NotImplementedError
-
-    def reorder_incremental_state(
-        self,
-        incremental_state: Dict[str, Dict[str, Optional[Tensor]]],
-        new_order: Tensor,
-    ):
-        """Reorder incremental state.
-
-        This will be called when the order of the input has changed from the
-        previous time step. A typical use case is beam search, where the input
-        order changes between time steps based on the selection of beams.
-        """
-        pass
-
-    def reorder_incremental_state_scripting(
-        self,
-        incremental_state: Dict[str, Dict[str, Optional[Tensor]]],
-        new_order: Tensor,
-    ):
-        """Main entry point for reordering the incremental state.
-
-        Due to limitations in TorchScript, we call this function in
-        :class:`fairseq.sequence_generator.SequenceGenerator` instead of
-        calling :func:`reorder_incremental_state` directly.
-        """
-        for module in self.modules():
-            if hasattr(module, "reorder_incremental_state"):
-                result = module.reorder_incremental_state(incremental_state, new_order)
-                if result is not None:
-                    incremental_state = result
-
-    def set_beam_size(self, beam_size):
-        """Sets the beam size in the decoder and all children."""
-        if getattr(self, "_beam_size", -1) != beam_size:
-            seen = set()
-
-            def apply_set_beam_size(module):
-                if (
-                    module != self
-                    and hasattr(module, "set_beam_size")
-                    and module not in seen
-                ):
-                    seen.add(module)
-                    module.set_beam_size(beam_size)
-
-            self.apply(apply_set_beam_size)
-            self._beam_size = beam_size

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/fairseq_model.py

@@ -1,573 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-"""
-Base classes for various fairseq models.
-"""
-
-import logging
-from argparse import Namespace
-from typing import Dict, List, Optional, Tuple
-
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-from fairseq import utils
-from fairseq.data import Dictionary
-from fairseq.dataclass.utils import (
-    convert_namespace_to_omegaconf,
-    gen_parser_from_dataclass,
-)
-from fairseq.models import FairseqDecoder, FairseqEncoder
-from omegaconf import DictConfig
-from torch import Tensor
-
-logger = logging.getLogger(__name__)
-
-
-def check_type(module, expected_type):
-    if hasattr(module, "unwrapped_module"):
-        assert isinstance(
-            module.unwrapped_module, expected_type
-        ), f"{type(module.unwrapped_module)} != {expected_type}"
-    else:
-        assert isinstance(module, expected_type), f"{type(module)} != {expected_type}"
-
-
-class BaseFairseqModel(nn.Module):
-    """Base class for fairseq models."""
-
-    def __init__(self):
-        super().__init__()
-        self._is_generation_fast = False
-
-    @classmethod
-    def add_args(cls, parser):
-        """Add model-specific arguments to the parser."""
-        dc = getattr(cls, "__dataclass", None)
-        if dc is not None:
-            # do not set defaults so that settings defaults from various architectures still works
-            gen_parser_from_dataclass(parser, dc(), delete_default=True)
-
-    @classmethod
-    def build_model(cls, args, task):
-        """Build a new model instance."""
-        raise NotImplementedError("Model must implement the build_model method")
-
-    def get_targets(self, sample, net_output):
-        """Get targets from either the sample or the net's output."""
-        return sample["target"]
-
-    def get_normalized_probs(
-        self,
-        net_output: Tuple[Tensor, Optional[Dict[str, List[Optional[Tensor]]]]],
-        log_probs: bool,
-        sample: Optional[Dict[str, Tensor]] = None,
-    ):
-        """Get normalized probabilities (or log probs) from a net's output."""
-        return self.get_normalized_probs_scriptable(net_output, log_probs, sample)
-
-    # TorchScript doesn't support super() method so that the scriptable Subclass
-    # can't access the base class model in Torchscript.
-    # Current workaround is to add a helper function with different name and
-    # call the helper function from scriptable Subclass.
-    def get_normalized_probs_scriptable(
-        self,
-        net_output: Tuple[Tensor, Optional[Dict[str, List[Optional[Tensor]]]]],
-        log_probs: bool,
-        sample: Optional[Dict[str, Tensor]] = None,
-    ):
-        """Scriptable helper function for get_normalized_probs in ~BaseFairseqModel"""
-        if hasattr(self, "decoder"):
-            return self.decoder.get_normalized_probs(net_output, log_probs, sample)
-        elif torch.is_tensor(net_output):
-            # syntactic sugar for simple models which don't have a decoder
-            # (e.g., the classification tutorial)
-            logits = net_output.float()
-            if log_probs:
-                return F.log_softmax(logits, dim=-1)
-            else:
-                return F.softmax(logits, dim=-1)
-        raise NotImplementedError
-
-    def extract_features(self, *args, **kwargs):
-        """Similar to *forward* but only return features."""
-        return self(*args, **kwargs)
-
-    def max_positions(self):
-        """Maximum length supported by the model."""
-        return None
-
-    def load_state_dict(
-        self,
-        state_dict,
-        strict=True,
-        model_cfg: Optional[DictConfig] = None,
-        args: Optional[Namespace] = None,
-    ):
-        """Copies parameters and buffers from *state_dict* into this module and
-        its descendants.
-
-        Overrides the method in :class:`nn.Module`. Compared with that method
-        this additionally "upgrades" *state_dicts* from old checkpoints.
-        """
-
-        if model_cfg is None and args is not None:
-            logger.warn(
-                "using 'args' is deprecated, please update your code to use dataclass config"
-            )
-            model_cfg = convert_namespace_to_omegaconf(args).model
-
-        self.upgrade_state_dict(state_dict)
-
-        from fairseq.checkpoint_utils import prune_state_dict
-
-        new_state_dict = prune_state_dict(state_dict, model_cfg)
-        return super().load_state_dict(new_state_dict, strict)
-
-    def upgrade_state_dict(self, state_dict):
-        """Upgrade old state dicts to work with newer code."""
-        self.upgrade_state_dict_named(state_dict, "")
-
-    def upgrade_state_dict_named(self, state_dict, name):
-        """Upgrade old state dicts to work with newer code.
-
-        Args:
-            state_dict (dict): state dictionary to upgrade, in place
-            name (str): the state dict key corresponding to the current module
-        """
-        assert state_dict is not None
-
-        def do_upgrade(m, prefix):
-            if len(prefix) > 0:
-                prefix += "."
-
-            for n, c in m.named_children():
-                name = prefix + n
-                if hasattr(c, "upgrade_state_dict_named"):
-                    c.upgrade_state_dict_named(state_dict, name)
-                elif hasattr(c, "upgrade_state_dict"):
-                    c.upgrade_state_dict(state_dict)
-                do_upgrade(c, name)
-
-        do_upgrade(self, name)
-
-    def set_num_updates(self, num_updates):
-        """State from trainer to pass along to model at every update."""
-        for m in self.modules():
-            if hasattr(m, "set_num_updates") and m != self:
-                m.set_num_updates(num_updates)
-
-    def prepare_for_inference_(self, cfg: DictConfig):
-        """Prepare model for inference."""
-        kwargs = {}
-        kwargs["beamable_mm_beam_size"] = (
-            None
-            if getattr(cfg.generation, "no_beamable_mm", False)
-            else getattr(cfg.generation, "beam", 5)
-        )
-        kwargs["need_attn"] = getattr(cfg.generation, "print_alignment", False)
-        if getattr(cfg.generation, "retain_dropout", False):
-            kwargs["retain_dropout"] = cfg.generation.retain_dropout
-            kwargs["retain_dropout_modules"] = cfg.generation.retain_dropout_modules
-        self.make_generation_fast_(**kwargs)
-
-    def make_generation_fast_(self, **kwargs):
-        """
-        Legacy entry point to optimize model for faster generation.
-        Prefer prepare_for_inference_.
-        """
-        if self._is_generation_fast:
-            return  # only apply once
-        self._is_generation_fast = True
-
-        # remove weight norm from all modules in the network
-        def apply_remove_weight_norm(module):
-            try:
-                nn.utils.remove_weight_norm(module)
-            except (AttributeError, ValueError):  # this module didn't have weight norm
-                return
-
-        self.apply(apply_remove_weight_norm)
-
-        def apply_make_generation_fast_(module, prefix):
-            if len(prefix) > 0:
-                prefix += "."
-
-            base_func = BaseFairseqModel.make_generation_fast_
-            for n, m in module.named_modules():
-                if (
-                    m != self
-                    and hasattr(m, "make_generation_fast_")
-                    # don't call this implementation again, e.g., if
-                    # children modules also inherit from BaseFairseqModel
-                    and m.make_generation_fast_.__func__ is not base_func
-                ):
-                    name = prefix + n
-                    m.make_generation_fast_(name=name, **kwargs)
-
-        apply_make_generation_fast_(self, "")
-
-        def train(mode=True):
-            if mode:
-                raise RuntimeError("cannot train after make_generation_fast")
-
-        # this model should no longer be used for training
-        self.eval()
-        self.train = train
-
-    def prepare_for_onnx_export_(self, **kwargs):
-        """Make model exportable via ONNX trace."""
-        seen = set()
-
-        def apply_prepare_for_onnx_export_(module):
-            if (
-                module != self
-                and hasattr(module, "prepare_for_onnx_export_")
-                and module not in seen
-            ):
-                seen.add(module)
-                module.prepare_for_onnx_export_(**kwargs)
-
-        self.apply(apply_prepare_for_onnx_export_)
-
-    @classmethod
-    def from_pretrained(
-        cls,
-        model_name_or_path,
-        checkpoint_file="model.pt",
-        data_name_or_path=".",
-        **kwargs,
-    ):
-        """
-        Load a :class:`~fairseq.models.FairseqModel` from a pre-trained model
-        file. Downloads and caches the pre-trained model file if needed.
-
-        The base implementation returns a
-        :class:`~fairseq.hub_utils.GeneratorHubInterface`, which can be used to
-        generate translations or sample from language models. The underlying
-        :class:`~fairseq.models.FairseqModel` can be accessed via the
-        *generator.models* attribute.
-
-        Other models may override this to implement custom hub interfaces.
-
-        Args:
-            model_name_or_path (str): either the name of a pre-trained model to
-                load or a path/URL to a pre-trained model state dict
-            checkpoint_file (str, optional): colon-separated list of checkpoint
-                files in the model archive to ensemble (default: 'model.pt')
-            data_name_or_path (str, optional): point args.data to the archive
-                at the given path/URL. Can start with '.' or './' to reuse the
-                model archive path.
-        """
-        from fairseq import hub_utils
-
-        x = hub_utils.from_pretrained(
-            model_name_or_path,
-            checkpoint_file,
-            data_name_or_path,
-            archive_map=cls.hub_models(),
-            **kwargs,
-        )
-        logger.info(x["args"])
-        return hub_utils.GeneratorHubInterface(x["args"], x["task"], x["models"])
-
-    @classmethod
-    def hub_models(cls):
-        return {}
-
-
-class FairseqEncoderDecoderModel(BaseFairseqModel):
-    """Base class for encoder-decoder models.
-
-    Args:
-        encoder (FairseqEncoder): the encoder
-        decoder (FairseqDecoder): the decoder
-    """
-
-    def __init__(self, encoder, decoder):
-        super().__init__()
-
-        self.encoder = encoder
-        self.decoder = decoder
-
-        check_type(self.encoder, FairseqEncoder)
-        check_type(self.decoder, FairseqDecoder)
-
-    def forward(self, src_tokens, src_lengths, prev_output_tokens, **kwargs):
-        """
-        Run the forward pass for an encoder-decoder model.
-
-        First feed a batch of source tokens through the encoder. Then, feed the
-        encoder output and previous decoder outputs (i.e., teacher forcing) to
-        the decoder to produce the next outputs::
-
-            encoder_out = self.encoder(src_tokens, src_lengths)
-            return self.decoder(prev_output_tokens, encoder_out)
-
-        Args:
-            src_tokens (LongTensor): tokens in the source language of shape
-                `(batch, src_len)`
-            src_lengths (LongTensor): source sentence lengths of shape `(batch)`
-            prev_output_tokens (LongTensor): previous decoder outputs of shape
-                `(batch, tgt_len)`, for teacher forcing
-
-        Returns:
-            tuple:
-                - the decoder's output of shape `(batch, tgt_len, vocab)`
-                - a dictionary with any model-specific outputs
-        """
-        encoder_out = self.encoder(src_tokens, src_lengths=src_lengths, **kwargs)
-        decoder_out = self.decoder(
-            prev_output_tokens, encoder_out=encoder_out, **kwargs
-        )
-        return decoder_out
-
-    def forward_decoder(self, prev_output_tokens, **kwargs):
-        return self.decoder(prev_output_tokens, **kwargs)
-
-    def extract_features(self, src_tokens, src_lengths, prev_output_tokens, **kwargs):
-        """
-        Similar to *forward* but only return features.
-
-        Returns:
-            tuple:
-                - the decoder's features of shape `(batch, tgt_len, embed_dim)`
-                - a dictionary with any model-specific outputs
-        """
-        encoder_out = self.encoder(src_tokens, src_lengths=src_lengths, **kwargs)
-        features = self.decoder.extract_features(
-            prev_output_tokens, encoder_out=encoder_out, **kwargs
-        )
-        return features
-
-    def output_layer(self, features, **kwargs):
-        """Project features to the default output size (typically vocabulary size)."""
-        return self.decoder.output_layer(features, **kwargs)
-
-    def max_positions(self):
-        """Maximum length supported by the model."""
-        return (self.encoder.max_positions(), self.decoder.max_positions())
-
-    def max_decoder_positions(self):
-        """Maximum length supported by the decoder."""
-        return self.decoder.max_positions()
-
-
-class FairseqModel(FairseqEncoderDecoderModel):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        utils.deprecation_warning(
-            "FairseqModel is deprecated, please use FairseqEncoderDecoderModel "
-            "or BaseFairseqModel instead",
-            stacklevel=4,
-        )
-
-
-class FairseqMultiModel(BaseFairseqModel):
-    """Base class for combining multiple encoder-decoder models."""
-
-    def __init__(self, encoders, decoders):
-        super().__init__()
-        assert encoders.keys() == decoders.keys()
-        self.keys = list(encoders.keys())
-        for key in self.keys:
-            check_type(encoders[key], FairseqEncoder)
-            check_type(decoders[key], FairseqDecoder)
-
-        self.models = nn.ModuleDict(
-            {
-                key: FairseqEncoderDecoderModel(encoders[key], decoders[key])
-                for key in self.keys
-            }
-        )
-
-    @staticmethod
-    def build_shared_embeddings(
-        dicts: Dict[str, Dictionary],
-        langs: List[str],
-        embed_dim: int,
-        build_embedding: callable,
-        pretrained_embed_path: Optional[str] = None,
-    ):
-        """
-        Helper function to build shared embeddings for a set of languages after
-        checking that all dicts corresponding to those languages are equivalent.
-
-        Args:
-            dicts: Dict of lang_id to its corresponding Dictionary
-            langs: languages that we want to share embeddings for
-            embed_dim: embedding dimension
-            build_embedding: callable function to actually build the embedding
-            pretrained_embed_path: Optional path to load pretrained embeddings
-        """
-        shared_dict = dicts[langs[0]]
-        if any(dicts[lang] != shared_dict for lang in langs):
-            raise ValueError(
-                "--share-*-embeddings requires a joined dictionary: "
-                "--share-encoder-embeddings requires a joined source "
-                "dictionary, --share-decoder-embeddings requires a joined "
-                "target dictionary, and --share-all-embeddings requires a "
-                "joint source + target dictionary."
-            )
-        return build_embedding(shared_dict, embed_dim, pretrained_embed_path)
-
-    def forward(self, src_tokens, src_lengths, prev_output_tokens, **kwargs):
-        raise NotImplementedError
-
-    def max_positions(self):
-        """Maximum length supported by the model."""
-        return {
-            key: (
-                self.models[key].encoder.max_positions(),
-                self.models[key].decoder.max_positions(),
-            )
-            for key in self.keys
-        }
-
-    def max_decoder_positions(self):
-        """Maximum length supported by the decoder."""
-        return min(model.decoder.max_positions() for model in self.models.values())
-
-    @property
-    def encoder(self):
-        return self.models[self.keys[0]].encoder
-
-    @property
-    def decoder(self):
-        return self.models[self.keys[0]].decoder
-
-    def forward_decoder(self, prev_output_tokens, **kwargs):
-        return self.decoder(prev_output_tokens, **kwargs)
-
-    def load_state_dict(
-        self,
-        state_dict,
-        strict=True,
-        model_cfg=None,
-        args: Optional[Namespace] = None,
-    ):
-        """Copies parameters and buffers from *state_dict* into this module and
-        its descendants.
-
-        Overrides the method in :class:`nn.Module`. Compared with that method
-        this additionally "upgrades" *state_dicts* from old checkpoints.
-        """
-
-        if model_cfg is None and args is not None:
-            logger.warn(
-                "using 'args' is deprecated, please update your code to use dataclass config"
-            )
-            model_cfg = convert_namespace_to_omegaconf(args).model
-
-        self.upgrade_state_dict(state_dict)
-
-        from fairseq.checkpoint_utils import prune_state_dict
-
-        new_state_dict = prune_state_dict(state_dict, model_cfg)
-        return super().load_state_dict(new_state_dict, strict)
-
-
-class FairseqLanguageModel(BaseFairseqModel):
-    """Base class for decoder-only models.
-
-    Args:
-        decoder (FairseqDecoder): the decoder
-    """
-
-    def __init__(self, decoder):
-        super().__init__()
-        self.decoder = decoder
-        check_type(self.decoder, FairseqDecoder)
-
-    def forward(self, src_tokens, **kwargs):
-        """
-        Run the forward pass for a decoder-only model.
-
-        Feeds a batch of tokens through the decoder to predict the next tokens.
-
-        Args:
-            src_tokens (LongTensor): tokens on which to condition the decoder,
-                of shape `(batch, tgt_len)`
-            src_lengths (LongTensor): source sentence lengths of shape `(batch)`
-
-        Returns:
-            tuple:
-                - the decoder's output of shape `(batch, seq_len, vocab)`
-                - a dictionary with any model-specific outputs
-        """
-        return self.decoder(src_tokens, **kwargs)
-
-    def forward_decoder(self, prev_output_tokens, **kwargs):
-        return self.decoder(prev_output_tokens, **kwargs)
-
-    def extract_features(self, src_tokens, **kwargs):
-        """
-        Similar to *forward* but only return features.
-
-        Returns:
-            tuple:
-                - the decoder's features of shape `(batch, seq_len, embed_dim)`
-                - a dictionary with any model-specific outputs
-        """
-        return self.decoder.extract_features(src_tokens, **kwargs)
-
-    def output_layer(self, features, **kwargs):
-        """Project features to the default output size (typically vocabulary size)."""
-        return self.decoder.output_layer(features, **kwargs)
-
-    def max_positions(self):
-        """Maximum length supported by the model."""
-        return self.decoder.max_positions()
-
-    def max_decoder_positions(self):
-        """Maximum length supported by the decoder."""
-        return self.decoder.max_positions()
-
-    @property
-    def supported_targets(self):
-        return {"future"}
-
-
-class FairseqEncoderModel(BaseFairseqModel):
-    """Base class for encoder-only models.
-
-    Args:
-        encoder (FairseqEncoder): the encoder
-    """
-
-    def __init__(self, encoder):
-        super().__init__()
-        self.encoder = encoder
-        check_type(self.encoder, FairseqEncoder)
-
-    def forward(self, src_tokens, src_lengths, **kwargs):
-        """
-        Run the forward pass for a encoder-only model.
-
-        Feeds a batch of tokens through the encoder to generate features.
-
-        Args:
-            src_tokens (LongTensor): input tokens of shape `(batch, src_len)`
-            src_lengths (LongTensor): source sentence lengths of shape `(batch)`
-
-        Returns:
-            the encoder's output, typically of shape `(batch, src_len, features)`
-        """
-        return self.encoder(src_tokens, src_lengths, **kwargs)
-
-    def get_normalized_probs(self, net_output, log_probs, sample=None):
-        """Get normalized probabilities (or log probs) from a net's output."""
-        encoder_out = net_output["encoder_out"]
-        if torch.is_tensor(encoder_out):
-            logits = encoder_out.float()
-            if log_probs:
-                return F.log_softmax(logits, dim=-1)
-            else:
-                return F.softmax(logits, dim=-1)
-        raise NotImplementedError
-
-    def max_positions(self):
-        """Maximum length supported by the model."""
-        return self.encoder.max_positions()

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/fconv.py

@@ -1,756 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-import math
-
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-from fairseq import utils
-from fairseq.models import (
-    FairseqEncoder,
-    FairseqEncoderDecoderModel,
-    FairseqIncrementalDecoder,
-    register_model,
-    register_model_architecture,
-)
-from fairseq.modules import (
-    AdaptiveSoftmax,
-    BeamableMM,
-    FairseqDropout,
-    GradMultiply,
-    LearnedPositionalEmbedding,
-    LinearizedConvolution,
-)
-
-
-@register_model("fconv")
-class FConvModel(FairseqEncoderDecoderModel):
-    """
-    A fully convolutional model, i.e. a convolutional encoder and a
-    convolutional decoder, as described in `"Convolutional Sequence to Sequence
-    Learning" (Gehring et al., 2017) <https://arxiv.org/abs/1705.03122>`_.
-
-    Args:
-        encoder (FConvEncoder): the encoder
-        decoder (FConvDecoder): the decoder
-
-    The Convolutional model provides the following named architectures and
-    command-line arguments:
-
-    .. argparse::
-        :ref: fairseq.models.fconv_parser
-        :prog:
-    """
-
-    @classmethod
-    def hub_models(cls):
-        def moses_subword(path):
-            return {
-                "path": path,
-                "tokenizer": "moses",
-                "bpe": "subword_nmt",
-            }
-
-        return {
-            "conv.wmt14.en-fr": moses_subword(
-                "https://dl.fbaipublicfiles.com/fairseq/models/wmt14.v2.en-fr.fconv-py.tar.bz2"
-            ),
-            "conv.wmt14.en-de": moses_subword(
-                "https://dl.fbaipublicfiles.com/fairseq/models/wmt14.en-de.fconv-py.tar.bz2"
-            ),
-            "conv.wmt17.en-de": moses_subword(
-                "https://dl.fbaipublicfiles.com/fairseq/models/wmt17.v2.en-de.fconv-py.tar.bz2"
-            ),
-        }
-
-    def __init__(self, encoder, decoder):
-        super().__init__(encoder, decoder)
-        self.encoder.num_attention_layers = sum(
-            layer is not None for layer in decoder.attention
-        )
-
-    @staticmethod
-    def add_args(parser):
-        """Add model-specific arguments to the parser."""
-        # fmt: off
-        parser.add_argument('--dropout', type=float, metavar='D',
-                            help='dropout probability')
-        parser.add_argument('--encoder-embed-dim', type=int, metavar='N',
-                            help='encoder embedding dimension')
-        parser.add_argument('--encoder-embed-path', type=str, metavar='STR',
-                            help='path to pre-trained encoder embedding')
-        parser.add_argument('--encoder-layers', type=str, metavar='EXPR',
-                            help='encoder layers [(dim, kernel_size), ...]')
-        parser.add_argument('--decoder-embed-dim', type=int, metavar='N',
-                            help='decoder embedding dimension')
-        parser.add_argument('--decoder-embed-path', type=str, metavar='STR',
-                            help='path to pre-trained decoder embedding')
-        parser.add_argument('--decoder-layers', type=str, metavar='EXPR',
-                            help='decoder layers [(dim, kernel_size), ...]')
-        parser.add_argument('--decoder-out-embed-dim', type=int, metavar='N',
-                            help='decoder output embedding dimension')
-        parser.add_argument('--decoder-attention', type=str, metavar='EXPR',
-                            help='decoder attention [True, ...]')
-        parser.add_argument('--share-input-output-embed', action='store_true',
-                            help='share input and output embeddings (requires'
-                                 ' --decoder-out-embed-dim and --decoder-embed-dim'
-                                 ' to be equal)')
-        # fmt: on
-
-    @classmethod
-    def build_model(cls, args, task):
-        """Build a new model instance."""
-        # make sure that all args are properly defaulted (in case there are any new ones)
-        base_architecture(args)
-
-        encoder_embed_dict = None
-        if args.encoder_embed_path:
-            encoder_embed_dict = utils.parse_embedding(args.encoder_embed_path)
-            utils.print_embed_overlap(encoder_embed_dict, task.source_dictionary)
-
-        decoder_embed_dict = None
-        if args.decoder_embed_path:
-            decoder_embed_dict = utils.parse_embedding(args.decoder_embed_path)
-            utils.print_embed_overlap(decoder_embed_dict, task.target_dictionary)
-
-        encoder = FConvEncoder(
-            dictionary=task.source_dictionary,
-            embed_dim=args.encoder_embed_dim,
-            embed_dict=encoder_embed_dict,
-            convolutions=eval(args.encoder_layers),
-            dropout=args.dropout,
-            max_positions=args.max_source_positions,
-        )
-        decoder = FConvDecoder(
-            dictionary=task.target_dictionary,
-            embed_dim=args.decoder_embed_dim,
-            embed_dict=decoder_embed_dict,
-            convolutions=eval(args.decoder_layers),
-            out_embed_dim=args.decoder_out_embed_dim,
-            attention=eval(args.decoder_attention),
-            dropout=args.dropout,
-            max_positions=args.max_target_positions,
-            share_embed=args.share_input_output_embed,
-        )
-        return FConvModel(encoder, decoder)
-
-
-class FConvEncoder(FairseqEncoder):
-    """
-    Convolutional encoder consisting of `len(convolutions)` layers.
-
-    Args:
-        dictionary (~fairseq.data.Dictionary): encoding dictionary
-        embed_dim (int, optional): embedding dimension
-        embed_dict (str, optional): filename from which to load pre-trained
-            embeddings
-        max_positions (int, optional): maximum supported input sequence length
-        convolutions (list, optional): the convolutional layer structure. Each
-            list item `i` corresponds to convolutional layer `i`. Layers are
-            given as ``(out_channels, kernel_width, [residual])``. Residual
-            connections are added between layers when ``residual=1`` (which is
-            the default behavior).
-        dropout (float, optional): dropout to be applied before each conv layer
-    """
-
-    def __init__(
-        self,
-        dictionary,
-        embed_dim=512,
-        embed_dict=None,
-        max_positions=1024,
-        convolutions=((512, 3),) * 20,
-        dropout=0.1,
-    ):
-        super().__init__(dictionary)
-        self.dropout_module = FairseqDropout(
-            dropout, module_name=self.__class__.__name__
-        )
-        self.num_attention_layers = None
-
-        num_embeddings = len(dictionary)
-        self.padding_idx = dictionary.pad()
-        self.embed_tokens = Embedding(num_embeddings, embed_dim, self.padding_idx)
-        if embed_dict:
-            self.embed_tokens = utils.load_embedding(
-                embed_dict, self.dictionary, self.embed_tokens
-            )
-
-        self.embed_positions = PositionalEmbedding(
-            max_positions,
-            embed_dim,
-            self.padding_idx,
-        )
-
-        convolutions = extend_conv_spec(convolutions)
-        in_channels = convolutions[0][0]
-        self.fc1 = Linear(embed_dim, in_channels, dropout=dropout)
-        self.projections = nn.ModuleList()
-        self.convolutions = nn.ModuleList()
-        self.residuals = []
-
-        layer_in_channels = [in_channels]
-        for _, (out_channels, kernel_size, residual) in enumerate(convolutions):
-            if residual == 0:
-                residual_dim = out_channels
-            else:
-                residual_dim = layer_in_channels[-residual]
-            self.projections.append(
-                Linear(residual_dim, out_channels)
-                if residual_dim != out_channels
-                else None
-            )
-            if kernel_size % 2 == 1:
-                padding = kernel_size // 2
-            else:
-                padding = 0
-            self.convolutions.append(
-                ConvTBC(
-                    in_channels,
-                    out_channels * 2,
-                    kernel_size,
-                    dropout=dropout,
-                    padding=padding,
-                )
-            )
-            self.residuals.append(residual)
-            in_channels = out_channels
-            layer_in_channels.append(out_channels)
-        self.fc2 = Linear(in_channels, embed_dim)
-
-    def forward(self, src_tokens, src_lengths):
-        """
-        Args:
-            src_tokens (LongTensor): tokens in the source language of shape
-                `(batch, src_len)`
-            src_lengths (LongTensor): lengths of each source sentence of shape
-                `(batch)`
-
-        Returns:
-            dict:
-                - **encoder_out** (tuple): a tuple with two elements, where the
-                  first element is the last encoder layer's output and the
-                  second element is the same quantity summed with the input
-                  embedding (used for attention). The shape of both tensors is
-                  `(batch, src_len, embed_dim)`.
-                - **encoder_padding_mask** (ByteTensor): the positions of
-                  padding elements of shape `(batch, src_len)`
-        """
-        # embed tokens and positions
-        x = self.embed_tokens(src_tokens) + self.embed_positions(src_tokens)
-        x = self.dropout_module(x)
-        input_embedding = x
-
-        # project to size of convolution
-        x = self.fc1(x)
-
-        # used to mask padding in input
-        encoder_padding_mask = src_tokens.eq(self.padding_idx).t()  # -> T x B
-        if not encoder_padding_mask.any():
-            encoder_padding_mask = None
-
-        # B x T x C -> T x B x C
-        x = x.transpose(0, 1)
-
-        residuals = [x]
-        # temporal convolutions
-        for proj, conv, res_layer in zip(
-            self.projections, self.convolutions, self.residuals
-        ):
-            if res_layer > 0:
-                residual = residuals[-res_layer]
-                residual = residual if proj is None else proj(residual)
-            else:
-                residual = None
-
-            if encoder_padding_mask is not None:
-                x = x.masked_fill(encoder_padding_mask.unsqueeze(-1), 0)
-
-            x = self.dropout_module(x)
-            if conv.kernel_size[0] % 2 == 1:
-                # padding is implicit in the conv
-                x = conv(x)
-            else:
-                padding_l = (conv.kernel_size[0] - 1) // 2
-                padding_r = conv.kernel_size[0] // 2
-                x = F.pad(x, (0, 0, 0, 0, padding_l, padding_r))
-                x = conv(x)
-            x = F.glu(x, dim=2)
-
-            if residual is not None:
-                x = (x + residual) * math.sqrt(0.5)
-            residuals.append(x)
-
-        # T x B x C -> B x T x C
-        x = x.transpose(1, 0)
-
-        # project back to size of embedding
-        x = self.fc2(x)
-
-        if encoder_padding_mask is not None:
-            encoder_padding_mask = encoder_padding_mask.t()  # -> B x T
-            x = x.masked_fill(encoder_padding_mask.unsqueeze(-1), 0)
-
-        # scale gradients (this only affects backward, not forward)
-        x = GradMultiply.apply(x, 1.0 / (2.0 * self.num_attention_layers))
-
-        # add output to input embedding for attention
-        y = (x + input_embedding) * math.sqrt(0.5)
-
-        return {
-            "encoder_out": (x, y),
-            "encoder_padding_mask": encoder_padding_mask,  # B x T
-        }
-
-    def reorder_encoder_out(self, encoder_out, new_order):
-        if encoder_out["encoder_out"] is not None:
-            encoder_out["encoder_out"] = (
-                encoder_out["encoder_out"][0].index_select(0, new_order),
-                encoder_out["encoder_out"][1].index_select(0, new_order),
-            )
-        if encoder_out["encoder_padding_mask"] is not None:
-            encoder_out["encoder_padding_mask"] = encoder_out[
-                "encoder_padding_mask"
-            ].index_select(0, new_order)
-        return encoder_out
-
-    def max_positions(self):
-        """Maximum input length supported by the encoder."""
-        return self.embed_positions.max_positions
-
-
-class AttentionLayer(nn.Module):
-    def __init__(self, conv_channels, embed_dim, bmm=None):
-        super().__init__()
-        # projects from output of convolution to embedding dimension
-        self.in_projection = Linear(conv_channels, embed_dim)
-        # projects from embedding dimension to convolution size
-        self.out_projection = Linear(embed_dim, conv_channels)
-
-        self.bmm = bmm if bmm is not None else torch.bmm
-
-    def forward(self, x, target_embedding, encoder_out, encoder_padding_mask):
-        residual = x
-
-        # attention
-        x = (self.in_projection(x) + target_embedding) * math.sqrt(0.5)
-        x = self.bmm(x, encoder_out[0])
-
-        # don't attend over padding
-        if encoder_padding_mask is not None:
-            x = (
-                x.float()
-                .masked_fill(encoder_padding_mask.unsqueeze(1), float("-inf"))
-                .type_as(x)
-            )  # FP16 support: cast to float and back
-
-        # softmax over last dim
-        sz = x.size()
-        x = F.softmax(x.view(sz[0] * sz[1], sz[2]), dim=1)
-        x = x.view(sz)
-        attn_scores = x
-
-        x = self.bmm(x, encoder_out[1])
-
-        # scale attention output (respecting potentially different lengths)
-        s = encoder_out[1].size(1)
-        if encoder_padding_mask is None:
-            x = x * (s * math.sqrt(1.0 / s))
-        else:
-            s = s - encoder_padding_mask.type_as(x).sum(
-                dim=1, keepdim=True
-            )  # exclude padding
-            s = s.unsqueeze(-1)
-            x = x * (s * s.rsqrt())
-
-        # project back
-        x = (self.out_projection(x) + residual) * math.sqrt(0.5)
-        return x, attn_scores
-
-    def make_generation_fast_(self, beamable_mm_beam_size=None, **kwargs):
-        """Replace torch.bmm with BeamableMM."""
-        if beamable_mm_beam_size is not None:
-            del self.bmm
-            self.add_module("bmm", BeamableMM(beamable_mm_beam_size))
-
-
-class FConvDecoder(FairseqIncrementalDecoder):
-    """Convolutional decoder"""
-
-    def __init__(
-        self,
-        dictionary,
-        embed_dim=512,
-        embed_dict=None,
-        out_embed_dim=256,
-        max_positions=1024,
-        convolutions=((512, 3),) * 20,
-        attention=True,
-        dropout=0.1,
-        share_embed=False,
-        positional_embeddings=True,
-        adaptive_softmax_cutoff=None,
-        adaptive_softmax_dropout=0.0,
-    ):
-        super().__init__(dictionary)
-        self.register_buffer("version", torch.Tensor([2]))
-        self.dropout_module = FairseqDropout(
-            dropout, module_name=self.__class__.__name__
-        )
-        self.need_attn = True
-
-        convolutions = extend_conv_spec(convolutions)
-        in_channels = convolutions[0][0]
-        if isinstance(attention, bool):
-            # expand True into [True, True, ...] and do the same with False
-            attention = [attention] * len(convolutions)
-        if not isinstance(attention, list) or len(attention) != len(convolutions):
-            raise ValueError(
-                "Attention is expected to be a list of booleans of "
-                "length equal to the number of layers."
-            )
-
-        num_embeddings = len(dictionary)
-        padding_idx = dictionary.pad()
-        self.embed_tokens = Embedding(num_embeddings, embed_dim, padding_idx)
-        if embed_dict:
-            self.embed_tokens = utils.load_embedding(
-                embed_dict, self.dictionary, self.embed_tokens
-            )
-
-        self.embed_positions = (
-            PositionalEmbedding(
-                max_positions,
-                embed_dim,
-                padding_idx,
-            )
-            if positional_embeddings
-            else None
-        )
-
-        self.fc1 = Linear(embed_dim, in_channels, dropout=dropout)
-        self.projections = nn.ModuleList()
-        self.convolutions = nn.ModuleList()
-        self.attention = nn.ModuleList()
-        self.residuals = []
-
-        layer_in_channels = [in_channels]
-        for i, (out_channels, kernel_size, residual) in enumerate(convolutions):
-            if residual == 0:
-                residual_dim = out_channels
-            else:
-                residual_dim = layer_in_channels[-residual]
-            self.projections.append(
-                Linear(residual_dim, out_channels)
-                if residual_dim != out_channels
-                else None
-            )
-            self.convolutions.append(
-                LinearizedConv1d(
-                    in_channels,
-                    out_channels * 2,
-                    kernel_size,
-                    padding=(kernel_size - 1),
-                    dropout=dropout,
-                )
-            )
-            self.attention.append(
-                AttentionLayer(out_channels, embed_dim) if attention[i] else None
-            )
-            self.residuals.append(residual)
-            in_channels = out_channels
-            layer_in_channels.append(out_channels)
-
-        self.adaptive_softmax = None
-        self.fc2 = self.fc3 = None
-
-        if adaptive_softmax_cutoff is not None:
-            assert not share_embed
-            self.adaptive_softmax = AdaptiveSoftmax(
-                num_embeddings,
-                in_channels,
-                adaptive_softmax_cutoff,
-                dropout=adaptive_softmax_dropout,
-            )
-        else:
-            self.fc2 = Linear(in_channels, out_embed_dim)
-            if share_embed:
-                assert out_embed_dim == embed_dim, (
-                    "Shared embed weights implies same dimensions "
-                    " out_embed_dim={} vs embed_dim={}".format(out_embed_dim, embed_dim)
-                )
-                self.fc3 = nn.Linear(out_embed_dim, num_embeddings)
-                self.fc3.weight = self.embed_tokens.weight
-            else:
-                self.fc3 = Linear(out_embed_dim, num_embeddings, dropout=dropout)
-
-    def forward(
-        self, prev_output_tokens, encoder_out=None, incremental_state=None, **unused
-    ):
-        if encoder_out is not None:
-            encoder_padding_mask = encoder_out["encoder_padding_mask"]
-            encoder_out = encoder_out["encoder_out"]
-
-            # split and transpose encoder outputs
-            encoder_a, encoder_b = self._split_encoder_out(
-                encoder_out, incremental_state
-            )
-
-        if self.embed_positions is not None:
-            pos_embed = self.embed_positions(prev_output_tokens, incremental_state)
-        else:
-            pos_embed = 0
-
-        if incremental_state is not None:
-            prev_output_tokens = prev_output_tokens[:, -1:]
-        x = self._embed_tokens(prev_output_tokens, incremental_state)
-
-        # embed tokens and combine with positional embeddings
-        x += pos_embed
-        x = self.dropout_module(x)
-        target_embedding = x
-
-        # project to size of convolution
-        x = self.fc1(x)
-
-        # B x T x C -> T x B x C
-        x = self._transpose_if_training(x, incremental_state)
-
-        # temporal convolutions
-        avg_attn_scores = None
-        num_attn_layers = len(self.attention)
-        residuals = [x]
-        for proj, conv, attention, res_layer in zip(
-            self.projections, self.convolutions, self.attention, self.residuals
-        ):
-            if res_layer > 0:
-                residual = residuals[-res_layer]
-                residual = residual if proj is None else proj(residual)
-            else:
-                residual = None
-
-            x = self.dropout_module(x)
-            x = conv(x, incremental_state)
-            x = F.glu(x, dim=2)
-
-            # attention
-            if attention is not None:
-                x = self._transpose_if_training(x, incremental_state)
-
-                x, attn_scores = attention(
-                    x, target_embedding, (encoder_a, encoder_b), encoder_padding_mask
-                )
-
-                if not self.training and self.need_attn:
-                    attn_scores = attn_scores / num_attn_layers
-                    if avg_attn_scores is None:
-                        avg_attn_scores = attn_scores
-                    else:
-                        avg_attn_scores.add_(attn_scores)
-
-                x = self._transpose_if_training(x, incremental_state)
-
-            # residual
-            if residual is not None:
-                x = (x + residual) * math.sqrt(0.5)
-            residuals.append(x)
-
-        # T x B x C -> B x T x C
-        x = self._transpose_if_training(x, incremental_state)
-
-        # project back to size of vocabulary if not using adaptive softmax
-        if self.fc2 is not None and self.fc3 is not None:
-            x = self.fc2(x)
-            x = self.dropout_module(x)
-            x = self.fc3(x)
-
-        return x, avg_attn_scores
-
-    def reorder_incremental_state(self, incremental_state, new_order):
-        super().reorder_incremental_state(incremental_state, new_order)
-        encoder_out = utils.get_incremental_state(
-            self, incremental_state, "encoder_out"
-        )
-        if encoder_out is not None:
-            encoder_out = tuple(eo.index_select(0, new_order) for eo in encoder_out)
-            utils.set_incremental_state(
-                self, incremental_state, "encoder_out", encoder_out
-            )
-
-    def max_positions(self):
-        """Maximum output length supported by the decoder."""
-        return (
-            self.embed_positions.max_positions
-            if self.embed_positions is not None
-            else float("inf")
-        )
-
-    def upgrade_state_dict(self, state_dict):
-        if utils.item(state_dict.get("decoder.version", torch.Tensor([1]))[0]) < 2:
-            # old models use incorrect weight norm dimension
-            for i, conv in enumerate(self.convolutions):
-                # reconfigure weight norm
-                nn.utils.remove_weight_norm(conv)
-                self.convolutions[i] = nn.utils.weight_norm(conv, dim=0)
-            state_dict["decoder.version"] = torch.Tensor([1])
-        return state_dict
-
-    def make_generation_fast_(self, need_attn=False, **kwargs):
-        self.need_attn = need_attn
-
-    def _embed_tokens(self, tokens, incremental_state):
-        if incremental_state is not None:
-            # keep only the last token for incremental forward pass
-            tokens = tokens[:, -1:]
-        return self.embed_tokens(tokens)
-
-    def _split_encoder_out(self, encoder_out, incremental_state):
-        """Split and transpose encoder outputs.
-
-        This is cached when doing incremental inference.
-        """
-        cached_result = utils.get_incremental_state(
-            self, incremental_state, "encoder_out"
-        )
-        if cached_result is not None:
-            return cached_result
-
-        # transpose only once to speed up attention layers
-        encoder_a, encoder_b = encoder_out
-        encoder_a = encoder_a.transpose(1, 2).contiguous()
-        result = (encoder_a, encoder_b)
-
-        if incremental_state is not None:
-            utils.set_incremental_state(self, incremental_state, "encoder_out", result)
-        return result
-
-    def _transpose_if_training(self, x, incremental_state):
-        if incremental_state is None:
-            x = x.transpose(0, 1)
-        return x
-
-
-def extend_conv_spec(convolutions):
-    """
-    Extends convolutional spec that is a list of tuples of 2 or 3 parameters
-    (kernel size, dim size and optionally how many layers behind to look for residual)
-    to default the residual propagation param if it is not specified
-    """
-    extended = []
-    for spec in convolutions:
-        if len(spec) == 3:
-            extended.append(spec)
-        elif len(spec) == 2:
-            extended.append(spec + (1,))
-        else:
-            raise Exception(
-                "invalid number of parameters in convolution spec "
-                + str(spec)
-                + ". expected 2 or 3"
-            )
-    return tuple(extended)
-
-
-def Embedding(num_embeddings, embedding_dim, padding_idx):
-    m = nn.Embedding(num_embeddings, embedding_dim, padding_idx=padding_idx)
-    nn.init.normal_(m.weight, 0, 0.1)
-    nn.init.constant_(m.weight[padding_idx], 0)
-    return m
-
-
-def PositionalEmbedding(num_embeddings, embedding_dim, padding_idx):
-    m = LearnedPositionalEmbedding(num_embeddings, embedding_dim, padding_idx)
-    nn.init.normal_(m.weight, 0, 0.1)
-    nn.init.constant_(m.weight[padding_idx], 0)
-    return m
-
-
-def Linear(in_features, out_features, dropout=0.0):
-    """Weight-normalized Linear layer (input: N x T x C)"""
-    m = nn.Linear(in_features, out_features)
-    nn.init.normal_(m.weight, mean=0, std=math.sqrt((1 - dropout) / in_features))
-    nn.init.constant_(m.bias, 0)
-    return nn.utils.weight_norm(m)
-
-
-def LinearizedConv1d(in_channels, out_channels, kernel_size, dropout=0.0, **kwargs):
-    """Weight-normalized Conv1d layer optimized for decoding"""
-    m = LinearizedConvolution(in_channels, out_channels, kernel_size, **kwargs)
-    std = math.sqrt((4 * (1.0 - dropout)) / (m.kernel_size[0] * in_channels))
-    nn.init.normal_(m.weight, mean=0, std=std)
-    nn.init.constant_(m.bias, 0)
-    return nn.utils.weight_norm(m, dim=2)
-
-
-def ConvTBC(in_channels, out_channels, kernel_size, dropout=0.0, **kwargs):
-    """Weight-normalized Conv1d layer"""
-    from fairseq.modules import ConvTBC
-
-    m = ConvTBC(in_channels, out_channels, kernel_size, **kwargs)
-    std = math.sqrt((4 * (1.0 - dropout)) / (m.kernel_size[0] * in_channels))
-    nn.init.normal_(m.weight, mean=0, std=std)
-    nn.init.constant_(m.bias, 0)
-    return nn.utils.weight_norm(m, dim=2)
-
-
-@register_model_architecture("fconv", "fconv")
-def base_architecture(args):
-    args.dropout = getattr(args, "dropout", 0.1)
-    args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 512)
-    args.encoder_embed_path = getattr(args, "encoder_embed_path", None)
-    args.encoder_layers = getattr(args, "encoder_layers", "[(512, 3)] * 20")
-    args.decoder_embed_dim = getattr(args, "decoder_embed_dim", 512)
-    args.decoder_embed_path = getattr(args, "decoder_embed_path", None)
-    args.decoder_layers = getattr(args, "decoder_layers", "[(512, 3)] * 20")
-    args.decoder_out_embed_dim = getattr(args, "decoder_out_embed_dim", 256)
-    args.decoder_attention = getattr(args, "decoder_attention", "True")
-    args.share_input_output_embed = getattr(args, "share_input_output_embed", False)
-
-
-@register_model_architecture("fconv", "fconv_iwslt_de_en")
-def fconv_iwslt_de_en(args):
-    args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 256)
-    args.encoder_layers = getattr(args, "encoder_layers", "[(256, 3)] * 4")
-    args.decoder_embed_dim = getattr(args, "decoder_embed_dim", 256)
-    args.decoder_layers = getattr(args, "decoder_layers", "[(256, 3)] * 3")
-    args.decoder_out_embed_dim = getattr(args, "decoder_out_embed_dim", 256)
-    base_architecture(args)
-
-
-@register_model_architecture("fconv", "fconv_wmt_en_ro")
-def fconv_wmt_en_ro(args):
-    args.decoder_out_embed_dim = getattr(args, "decoder_out_embed_dim", 512)
-    base_architecture(args)
-
-
-@register_model_architecture("fconv", "fconv_wmt_en_de")
-def fconv_wmt_en_de(args):
-    convs = "[(512, 3)] * 9"  # first 9 layers have 512 units
-    convs += " + [(1024, 3)] * 4"  # next 4 layers have 1024 units
-    convs += " + [(2048, 1)] * 2"  # final 2 layers use 1x1 convolutions
-
-    args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 768)
-    args.encoder_layers = getattr(args, "encoder_layers", convs)
-    args.decoder_embed_dim = getattr(args, "decoder_embed_dim", 768)
-    args.decoder_layers = getattr(args, "decoder_layers", convs)
-    args.decoder_out_embed_dim = getattr(args, "decoder_out_embed_dim", 512)
-    base_architecture(args)
-
-
-@register_model_architecture("fconv", "fconv_wmt_en_fr")
-def fconv_wmt_en_fr(args):
-    convs = "[(512, 3)] * 6"  # first 6 layers have 512 units
-    convs += " + [(768, 3)] * 4"  # next 4 layers have 768 units
-    convs += " + [(1024, 3)] * 3"  # next 3 layers have 1024 units
-    convs += " + [(2048, 1)] * 1"  # next 1 layer uses 1x1 convolutions
-    convs += " + [(4096, 1)] * 1"  # final 1 layer uses 1x1 convolutions
-
-    args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 768)
-    args.encoder_layers = getattr(args, "encoder_layers", convs)
-    args.decoder_embed_dim = getattr(args, "decoder_embed_dim", 768)
-    args.decoder_layers = getattr(args, "decoder_layers", convs)
-    args.decoder_out_embed_dim = getattr(args, "decoder_out_embed_dim", 512)
-    base_architecture(args)

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/fconv_lm.py

@@ -1,136 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-from fairseq import utils
-from fairseq.models import (
-    FairseqLanguageModel,
-    register_model,
-    register_model_architecture,
-)
-from fairseq.models.fconv import FConvDecoder
-from fairseq.utils import safe_hasattr
-
-
-@register_model("fconv_lm")
-class FConvLanguageModel(FairseqLanguageModel):
-    def __init__(self, decoder):
-        super().__init__(decoder)
-
-    @staticmethod
-    def add_args(parser):
-        """Add model-specific arguments to the parser."""
-        parser.add_argument(
-            "--dropout", type=float, metavar="D", help="dropout probability"
-        )
-        parser.add_argument(
-            "--decoder-embed-dim",
-            type=int,
-            metavar="N",
-            help="decoder embedding dimension",
-        )
-        parser.add_argument(
-            "--decoder-layers",
-            type=str,
-            metavar="EXPR",
-            help="decoder layers [(dim, kernel_size), ...]",
-        )
-        parser.add_argument(
-            "--decoder-out-embed-dim",
-            type=int,
-            metavar="N",
-            help="decoder output embedding dimension",
-        )
-        parser.add_argument(
-            "--adaptive-softmax-cutoff",
-            metavar="EXPR",
-            help="comma separated list of adaptive softmax cutoff points. "
-            "Must be used with adaptive_loss criterion",
-        )
-        parser.add_argument(
-            "--adaptive-softmax-dropout",
-            type=float,
-            metavar="D",
-            help="sets adaptive softmax dropout for the tail projections",
-        )
-        parser.add_argument(
-            "--decoder-attention",
-            type=str,
-            metavar="EXPR",
-            help="decoder attention [True, ...]",
-        )
-
-    @classmethod
-    def build_model(cls, args, task):
-        """Build a new model instance."""
-        # make sure all arguments are present in older models
-        base_lm_architecture(args)
-
-        if safe_hasattr(args, "max_target_positions") and not safe_hasattr(
-            args, "tokens_per_sample"
-        ):
-            args.tokens_per_sample = args.max_target_positions
-
-        decoder = FConvDecoder(
-            dictionary=task.target_dictionary,
-            embed_dim=args.decoder_embed_dim,
-            convolutions=eval(args.decoder_layers),
-            out_embed_dim=args.decoder_embed_dim,
-            attention=eval(args.decoder_attention),
-            dropout=args.dropout,
-            max_positions=args.tokens_per_sample,
-            share_embed=False,
-            positional_embeddings=False,
-            adaptive_softmax_cutoff=(
-                utils.eval_str_list(args.adaptive_softmax_cutoff, type=int)
-                if args.criterion == "adaptive_loss"
-                else None
-            ),
-            adaptive_softmax_dropout=args.adaptive_softmax_dropout,
-        )
-        return FConvLanguageModel(decoder)
-
-
-@register_model_architecture("fconv_lm", "fconv_lm")
-def base_lm_architecture(args):
-    args.dropout = getattr(args, "dropout", 0.1)
-    args.decoder_embed_dim = getattr(args, "decoder_embed_dim", 128)
-    args.decoder_layers = getattr(args, "decoder_layers", "[(1268, 4)] * 13")
-    args.decoder_attention = getattr(args, "decoder_attention", "False")
-    args.adaptive_softmax_cutoff = getattr(args, "adaptive_softmax_cutoff", None)
-    args.adaptive_softmax_dropout = getattr(args, "adaptive_softmax_dropout", 0)
-
-
-@register_model_architecture("fconv_lm", "fconv_lm_dauphin_wikitext103")
-def fconv_lm_dauphin_wikitext103(args):
-    layers = "[(850, 6)] * 3"
-    layers += " + [(850, 1)] * 1"
-    layers += " + [(850, 5)] * 4"
-    layers += " + [(850, 1)] * 1"
-    layers += " + [(850, 4)] * 3"
-    layers += " + [(1024, 4)] * 1"
-    layers += " + [(2048, 4)] * 1"
-    args.decoder_embed_dim = getattr(args, "decoder_embed_dim", 280)
-    args.decoder_layers = getattr(args, "decoder_layers", layers)
-    args.decoder_attention = getattr(args, "decoder_attention", "False")
-    args.adaptive_softmax_cutoff = getattr(
-        args, "adaptive_softmax_cutoff", "10000,20000,200000"
-    )
-    base_lm_architecture(args)
-
-
-@register_model_architecture("fconv_lm", "fconv_lm_dauphin_gbw")
-def fconv_lm_dauphin_gbw(args):
-    layers = "[(512, 5)]"
-    layers += " + [(128, 1, 0), (128, 5, 0), (512, 1, 3)] * 3"
-    layers += " + [(512, 1, 0), (512, 5, 0), (1024, 1, 3)] * 3"
-    layers += " + [(1024, 1, 0), (1024, 5, 0), (2048, 1, 3)] * 6"
-    layers += " + [(1024, 1, 0), (1024, 5, 0), (4096, 1, 3)]"
-    args.decoder_embed_dim = getattr(args, "decoder_embed_dim", 128)
-    args.decoder_layers = getattr(args, "decoder_layers", layers)
-    args.decoder_attention = getattr(args, "decoder_attention", "False")
-    args.adaptive_softmax_cutoff = getattr(
-        args, "adaptive_softmax_cutoff", "10000,50000,200000"
-    )
-    base_lm_architecture(args)

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/fconv_self_att.py

@@ -1,673 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-import logging
-import math
-import os
-
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-from fairseq import checkpoint_utils
-from fairseq.incremental_decoding_utils import with_incremental_state
-from fairseq.models import (
-    CompositeEncoder,
-    FairseqDecoder,
-    FairseqEncoder,
-    FairseqEncoderDecoderModel,
-    register_model,
-    register_model_architecture,
-)
-from fairseq.modules import (
-    DownsampledMultiHeadAttention,
-    FairseqDropout,
-    GradMultiply,
-    LayerNorm,
-    LearnedPositionalEmbedding,
-    LinearizedConvolution,
-)
-
-logger = logging.getLogger(__name__)
-
-
-@register_model("fconv_self_att")
-class FConvModelSelfAtt(FairseqEncoderDecoderModel):
-    @classmethod
-    def hub_models(cls):
-        return {
-            "conv.stories.pretrained": {
-                "path": "https://dl.fbaipublicfiles.com/fairseq/models/stories_checkpoint.tar.gz",
-                "checkpoint_file": "pretrained_checkpoint.pt",
-                "tokenizer": "nltk",
-            },
-            "conv.stories": {
-                "path": "https://dl.fbaipublicfiles.com/fairseq/models/stories_checkpoint.tar.gz",
-                "checkpoint_file": "fusion_checkpoint.pt",
-                "tokenizer": "nltk",
-                "pretrained": "True",
-                "pretrained_checkpoint": "./pretrained_checkpoint.pt",
-            },
-            # Test set containing dictionaries
-            "data.stories": "https://dl.fbaipublicfiles.com/fairseq/data/stories_test.tar.bz2",
-        }
-
-    def __init__(self, encoder, decoder, pretrained_encoder=None):
-        super().__init__(encoder, decoder)
-        self.encoder.num_attention_layers = sum(
-            layer is not None for layer in decoder.attention
-        )
-        self.pretrained_encoder = pretrained_encoder
-        if self.pretrained_encoder is None:
-            encoders = {"encoder": encoder}
-        else:
-            encoders = {"encoder": encoder, "pretrained": self.pretrained_encoder}
-        # for fusion model, CompositeEncoder contains both pretrained and training encoders
-        # these are forwarded and then combined in the decoder
-        self.encoder = CompositeEncoder(encoders)
-
-    @staticmethod
-    def add_args(parser):
-        """Add model-specific arguments to the parser."""
-        # fmt: off
-        parser.add_argument('--dropout', type=float, metavar='D',
-                            help='dropout probability')
-        parser.add_argument('--encoder-embed-dim', type=int, metavar='N',
-                            help='encoder embedding dimension')
-        parser.add_argument('--encoder-layers', type=str, metavar='EXPR',
-                            help='encoder layers [(dim, kernel_size), ...]')
-        parser.add_argument('--decoder-embed-dim', type=int, metavar='N',
-                            help='decoder embedding dimension')
-        parser.add_argument('--decoder-layers', type=str, metavar='EXPR',
-                            help='decoder layers [(dim, kernel_size), ...]')
-        parser.add_argument('--decoder-out-embed-dim', type=int, metavar='N',
-                            help='decoder output embedding dimension')
-        parser.add_argument('--decoder-attention', type=str, metavar='EXPR',
-                            help='decoder attention [True, ...]')
-        parser.add_argument('--self-attention', type=str, metavar='EXPR',
-                            help='decoder self-attention layers, ex: [True] + [False]*5')
-        parser.add_argument('--multihead-attention-nheads', type=int,
-                            help='Number of heads to use in attention')
-        parser.add_argument('--multihead-self-attention-nheads', type=int,
-                            help='Number of heads to use in self-attention')
-        parser.add_argument('--encoder-attention', type=str, metavar='EXPR',
-                            help='encoder attention [True, ...]')
-        parser.add_argument('--encoder-attention-nheads', type=int,
-                            help='Number of heads to use in encoder attention')
-        parser.add_argument('--project-input', type=str, metavar='EXPR',
-                            help='Use projections in self-attention [True, ...]')
-        parser.add_argument('--gated-attention', type=str, metavar='EXPR',
-                            help='Use GLU layers in self-attention projections [True, ...]')
-        parser.add_argument('--downsample', type=str, metavar='EXPR',
-                            help='Use downsampling in self-attention [True, ...]')
-        parser.add_argument('--pretrained-checkpoint', metavar='DIR',
-                            help='path to load checkpoint from pretrained model')
-        parser.add_argument('--pretrained', type=str, metavar='EXPR',
-                            help='use pretrained model when training [True, ...]')
-        # fmt: on
-
-    @classmethod
-    def build_model(cls, args, task):
-        """Build a new model instance."""
-        trained_encoder, trained_decoder = None, None
-        pretrained = eval(args.pretrained)
-        if pretrained:
-            logger.info("loading pretrained model")
-            if not os.path.exists(args.pretrained_checkpoint):
-                new_pretrained_checkpoint = os.path.join(
-                    args.data, args.pretrained_checkpoint
-                )
-                if os.path.exists(new_pretrained_checkpoint):
-                    args.pretrained_checkpoint = new_pretrained_checkpoint
-            trained_model = checkpoint_utils.load_model_ensemble(
-                filenames=[args.pretrained_checkpoint],
-                task=task,
-            )[0][0]
-            trained_decoder = list(trained_model.children())[1]
-            trained_encoder = list(trained_model.children())[0]
-
-            # freeze pretrained model
-            for param in trained_decoder.parameters():
-                param.requires_grad = False
-            for param in trained_encoder.parameters():
-                param.requires_grad = False
-
-        encoder = FConvEncoder(
-            task.source_dictionary,
-            embed_dim=args.encoder_embed_dim,
-            convolutions=eval(args.encoder_layers),
-            dropout=args.dropout,
-            max_positions=args.max_source_positions,
-            attention=eval(args.encoder_attention),
-            attention_nheads=args.encoder_attention_nheads,
-        )
-
-        decoder = FConvDecoder(
-            task.target_dictionary,
-            embed_dim=args.decoder_embed_dim,
-            convolutions=eval(args.decoder_layers),
-            out_embed_dim=args.decoder_out_embed_dim,
-            attention=eval(args.decoder_attention),
-            dropout=args.dropout,
-            max_positions=args.max_target_positions,
-            selfattention=eval(args.self_attention),
-            attention_nheads=args.multihead_attention_nheads,
-            selfattention_nheads=args.multihead_self_attention_nheads,
-            project_input=eval(args.project_input),
-            gated_attention=eval(args.gated_attention),
-            downsample=eval(args.downsample),
-            pretrained=pretrained,
-            trained_decoder=trained_decoder,
-        )
-        model = FConvModelSelfAtt(encoder, decoder, trained_encoder)
-
-        return model
-
-    @property
-    def pretrained(self):
-        return self.pretrained_encoder is not None
-
-
-class FConvEncoder(FairseqEncoder):
-    """Convolutional encoder"""
-
-    def __init__(
-        self,
-        dictionary,
-        embed_dim=512,
-        max_positions=1024,
-        convolutions=((512, 3),) * 20,
-        dropout=0.1,
-        attention=False,
-        attention_nheads=1,
-    ):
-        super().__init__(dictionary)
-        self.dropout_module = FairseqDropout(
-            dropout, module_name=self.__class__.__name__
-        )
-        self.num_attention_layers = None
-
-        num_embeddings = len(dictionary)
-        self.padding_idx = dictionary.pad()
-        self.embed_tokens = Embedding(num_embeddings, embed_dim, self.padding_idx)
-        self.embed_positions = PositionalEmbedding(
-            max_positions,
-            embed_dim,
-            self.padding_idx,
-        )
-
-        def expand_bool_array(val):
-            if isinstance(val, bool):
-                # expand True into [True, True, ...] and do the same with False
-                return [val] * len(convolutions)
-            return val
-
-        attention = expand_bool_array(attention)
-
-        in_channels = convolutions[0][0]
-        self.fc1 = Linear(embed_dim, in_channels, dropout=dropout)
-        self.projections = nn.ModuleList()
-        self.convolutions = nn.ModuleList()
-        self.attention = nn.ModuleList()
-        self.attproj = nn.ModuleList()
-        for i, (out_channels, kernel_size) in enumerate(convolutions):
-            self.projections.append(
-                Linear(in_channels, out_channels)
-                if in_channels != out_channels
-                else None
-            )
-            self.convolutions.append(
-                ConvTBC(in_channels, out_channels * 2, kernel_size, dropout=dropout)
-            )
-
-            self.attention.append(
-                SelfAttention(out_channels, embed_dim, attention_nheads)
-                if attention[i]
-                else None
-            )
-            in_channels = out_channels
-
-        self.fc2 = Linear(in_channels, embed_dim)
-
-    def forward(self, src_tokens, src_lengths):
-        # embed tokens and positions
-        x = self.embed_tokens(src_tokens) + self.embed_positions(src_tokens)
-        x = self.dropout_module(x)
-        input_embedding = x.transpose(0, 1)
-
-        # project to size of convolution
-        x = self.fc1(x)
-
-        encoder_padding_mask = src_tokens.eq(self.padding_idx).t()  # -> T x B
-        if not encoder_padding_mask.any():
-            encoder_padding_mask = None
-
-        # B x T x C -> T x B x C
-        x = x.transpose(0, 1)
-
-        # temporal convolutions
-        for proj, conv, attention in zip(
-            self.projections, self.convolutions, self.attention
-        ):
-            residual = x if proj is None else proj(x)
-
-            if encoder_padding_mask is not None:
-                x = x.masked_fill(encoder_padding_mask.unsqueeze(-1), 0)
-
-            x = self.dropout_module(x)
-            padding_l = (conv.kernel_size[0] - 1) // 2
-            padding_r = conv.kernel_size[0] // 2
-            x = F.pad(x, (0, 0, 0, 0, padding_l, padding_r))
-            x = conv(x)
-            x = F.glu(x, dim=2)
-            if attention is not None:
-                x = attention(x)
-            x = (x + residual) * math.sqrt(0.5)
-
-        # T x B x C -> B x T x C
-        x = x.transpose(1, 0)
-
-        # project back to size of embedding
-        x = self.fc2(x)
-
-        if encoder_padding_mask is not None:
-            encoder_padding_mask = encoder_padding_mask.t()  # -> B x T
-            x = x.masked_fill(encoder_padding_mask.unsqueeze(-1), 0)
-
-        # scale gradients (this only affects backward, not forward)
-        x = GradMultiply.apply(x, 1.0 / (2.0 * self.num_attention_layers))
-
-        # add output to input embedding for attention
-        y = (x + input_embedding.transpose(0, 1)) * math.sqrt(0.5)
-
-        return {
-            "encoder_out": (x, y),
-            "encoder_padding_mask": encoder_padding_mask,  # B x T
-        }
-
-    def reorder_encoder_out(self, encoder_out, new_order):
-        encoder_out["encoder_out"] = tuple(
-            eo.index_select(0, new_order) for eo in encoder_out["encoder_out"]
-        )
-
-        if encoder_out["encoder_padding_mask"] is not None:
-            encoder_out["encoder_padding_mask"] = encoder_out[
-                "encoder_padding_mask"
-            ].index_select(0, new_order)
-
-        if "pretrained" in encoder_out:
-            encoder_out["pretrained"]["encoder_out"] = tuple(
-                eo.index_select(0, new_order)
-                for eo in encoder_out["pretrained"]["encoder_out"]
-            )
-
-        return encoder_out
-
-    def max_positions(self):
-        """Maximum input length supported by the encoder."""
-        return self.embed_positions.max_positions
-
-
-@with_incremental_state
-class FConvDecoder(FairseqDecoder):
-    """Convolutional decoder"""
-
-    def __init__(
-        self,
-        dictionary,
-        embed_dim=512,
-        out_embed_dim=256,
-        max_positions=1024,
-        convolutions=((512, 3),) * 8,
-        attention=True,
-        dropout=0.1,
-        selfattention=False,
-        attention_nheads=1,
-        selfattention_nheads=1,
-        project_input=False,
-        gated_attention=False,
-        downsample=False,
-        pretrained=False,
-        trained_decoder=None,
-    ):
-        super().__init__(dictionary)
-        self.register_buffer("version", torch.Tensor([2]))
-        self.pretrained = pretrained
-        self.pretrained_decoder = trained_decoder
-        self.dropout_module = FairseqDropout(
-            dropout, module_name=self.__class__.__name__
-        )
-        self.need_attn = True
-        in_channels = convolutions[0][0]
-
-        def expand_bool_array(val):
-            if isinstance(val, bool):
-                # expand True into [True, True, ...] and do the same with False
-                return [val] * len(convolutions)
-            return val
-
-        attention = expand_bool_array(attention)
-        selfattention = expand_bool_array(selfattention)
-
-        if not isinstance(attention, list) or len(attention) != len(convolutions):
-            raise ValueError(
-                "Attention is expected to be a list of booleans of "
-                "length equal to the number of layers."
-            )
-
-        num_embeddings = len(dictionary)
-        padding_idx = dictionary.pad()
-        self.embed_tokens = Embedding(num_embeddings, embed_dim, padding_idx)
-
-        self.embed_positions = PositionalEmbedding(
-            max_positions,
-            embed_dim,
-            padding_idx,
-        )
-
-        self.fc1 = Linear(embed_dim, in_channels, dropout=dropout)
-        self.projections = nn.ModuleList()
-        self.convolutions = nn.ModuleList()
-        self.attention = nn.ModuleList()
-        self.selfattention = nn.ModuleList()
-        self.attproj = nn.ModuleList()
-        for i, (out_channels, kernel_size) in enumerate(convolutions):
-            self.projections.append(
-                Linear(in_channels, out_channels)
-                if in_channels != out_channels
-                else None
-            )
-            self.convolutions.append(
-                LinearizedConv1d(
-                    in_channels,
-                    out_channels * 2,
-                    kernel_size,
-                    padding=(kernel_size - 1),
-                    dropout=dropout,
-                )
-            )
-
-            self.attention.append(
-                DownsampledMultiHeadAttention(
-                    out_channels,
-                    embed_dim,
-                    attention_nheads,
-                    project_input=project_input,
-                    gated=False,
-                    downsample=False,
-                )
-                if attention[i]
-                else None
-            )
-
-            self.attproj.append(
-                Linear(out_channels, embed_dim, dropout=dropout)
-                if attention[i]
-                else None
-            )
-            self.selfattention.append(
-                SelfAttention(
-                    out_channels,
-                    embed_dim,
-                    selfattention_nheads,
-                    project_input=project_input,
-                    gated=gated_attention,
-                    downsample=downsample,
-                )
-                if selfattention[i]
-                else None
-            )
-            in_channels = out_channels
-
-        self.fc2 = Linear(in_channels, out_embed_dim)
-        self.fc3 = Linear(out_embed_dim, num_embeddings, dropout=dropout)
-
-        # model fusion
-        if self.pretrained:
-            # independent gates are learned from the concatenated input
-            self.gate1 = nn.Sequential(
-                Linear(out_embed_dim * 2, out_embed_dim), nn.Sigmoid()
-            )
-            self.gate2 = nn.Sequential(
-                Linear(out_embed_dim * 2, out_embed_dim), nn.Sigmoid()
-            )
-            # pretrained and trained models are joined
-            self.joining = nn.Sequential(
-                Linear(out_embed_dim * 2, out_embed_dim * 2),
-                LayerNorm(out_embed_dim * 2),
-                nn.GLU(),
-                Linear(out_embed_dim, out_embed_dim * 2),
-                LayerNorm(out_embed_dim * 2),
-                nn.GLU(),
-                Linear(out_embed_dim, out_embed_dim),
-                LayerNorm(out_embed_dim),
-            )
-            # pretrained model contains an output layer that is nhid -> vocab size
-            # but the models are combined in their hidden state
-            # the hook stores the output of the pretrained model forward
-            self.pretrained_outputs = {}
-
-            def save_output():
-                def hook(a, b, output):
-                    self.pretrained_outputs["out"] = output
-
-                return hook
-
-            self.pretrained_decoder.fc2.register_forward_hook(save_output())
-
-    def forward(self, prev_output_tokens, encoder_out):
-        trained_encoder_out = encoder_out["pretrained"] if self.pretrained else None
-        encoder_out = encoder_out["encoder"]["encoder_out"]
-
-        encoder_a, encoder_b = self._split_encoder_out(encoder_out)
-
-        # embed positions
-        positions = self.embed_positions(prev_output_tokens)
-
-        # embed tokens and positions
-        x = self.embed_tokens(prev_output_tokens) + positions
-        x = self.dropout_module(x)
-        target_embedding = x.transpose(0, 1)
-
-        # project to size of convolution
-        x = self.fc1(x)
-
-        # B x T x C -> T x B x C
-        x = x.transpose(0, 1)
-
-        # temporal convolutions
-        avg_attn_scores = None
-        for proj, conv, attention, selfattention, attproj in zip(
-            self.projections,
-            self.convolutions,
-            self.attention,
-            self.selfattention,
-            self.attproj,
-        ):
-            residual = x if proj is None else proj(x)
-
-            x = self.dropout_module(x)
-            x = conv(x)
-            x = F.glu(x, dim=2)
-
-            # attention
-            if attention is not None:
-                r = x
-                x, attn_scores = attention(
-                    attproj(x) + target_embedding, encoder_a, encoder_b
-                )
-                x = x + r
-                if not self.training and self.need_attn:
-                    if avg_attn_scores is None:
-                        avg_attn_scores = attn_scores
-                    else:
-                        avg_attn_scores.add_(attn_scores)
-
-            if selfattention is not None:
-                x = selfattention(x)
-
-            x = (x + residual) * math.sqrt(0.5)
-
-        # T x B x C -> B x T x C
-        x = x.transpose(0, 1)
-
-        # project back to size of vocabulary
-        x = self.fc2(x)
-        x = self.dropout_module(x)
-        if not self.pretrained:
-            x = self.fc3(x)
-
-        # fusion gating
-        if self.pretrained:
-            trained_x, _ = self.pretrained_decoder.forward(
-                prev_output_tokens, trained_encoder_out
-            )
-            y = torch.cat([x, self.pretrained_outputs["out"]], dim=-1)
-            gate1 = self.gate1(y)
-            gate2 = self.gate2(y)
-            gated_x1 = gate1 * x
-            gated_x2 = gate2 * self.pretrained_outputs["out"]
-            fusion = torch.cat([gated_x1, gated_x2], dim=-1)
-            fusion = self.joining(fusion)
-            fusion_output = self.fc3(fusion)
-            return fusion_output, avg_attn_scores
-        else:
-            return x, avg_attn_scores
-
-    def max_positions(self):
-        """Maximum output length supported by the decoder."""
-        return self.embed_positions.max_positions
-
-    def make_generation_fast_(self, need_attn=False, **kwargs):
-        self.need_attn = need_attn
-
-    def _split_encoder_out(self, encoder_out):
-        """Split and transpose encoder outputs."""
-        # transpose only once to speed up attention layers
-        encoder_a, encoder_b = encoder_out
-        encoder_a = encoder_a.transpose(0, 1).contiguous()
-        encoder_b = encoder_b.transpose(0, 1).contiguous()
-        result = (encoder_a, encoder_b)
-        return result
-
-
-class SelfAttention(nn.Module):
-    def __init__(
-        self,
-        out_channels,
-        embed_dim,
-        num_heads,
-        project_input=False,
-        gated=False,
-        downsample=False,
-    ):
-        super().__init__()
-        self.attention = DownsampledMultiHeadAttention(
-            out_channels,
-            embed_dim,
-            num_heads,
-            dropout=0,
-            bias=True,
-            project_input=project_input,
-            gated=gated,
-            downsample=downsample,
-        )
-        self.in_proj_q = Linear(out_channels, embed_dim)
-        self.in_proj_k = Linear(out_channels, embed_dim)
-        self.in_proj_v = Linear(out_channels, embed_dim)
-        self.ln = LayerNorm(out_channels)
-
-    def forward(self, x):
-        residual = x
-        query = self.in_proj_q(x)
-        key = self.in_proj_k(x)
-        value = self.in_proj_v(x)
-        x, _ = self.attention(
-            query, key, value, mask_future_timesteps=True, use_scalar_bias=True
-        )
-        return self.ln(x + residual)
-
-
-def Embedding(num_embeddings, embedding_dim, padding_idx):
-    m = nn.Embedding(num_embeddings, embedding_dim, padding_idx=padding_idx)
-    m.weight.data.normal_(0, 0.1)
-    return m
-
-
-def PositionalEmbedding(num_embeddings, embedding_dim, padding_idx):
-    m = LearnedPositionalEmbedding(num_embeddings, embedding_dim, padding_idx)
-    m.weight.data.normal_(0, 0.1)
-    return m
-
-
-def Linear(in_features, out_features, dropout=0.0):
-    """Weight-normalized Linear layer (input: N x T x C)"""
-    m = nn.Linear(in_features, out_features)
-    m.weight.data.normal_(mean=0, std=math.sqrt((1 - dropout) / in_features))
-    m.bias.data.zero_()
-    return m
-
-
-def LinearizedConv1d(in_channels, out_channels, kernel_size, dropout=0.0, **kwargs):
-    """Weight-normalized Conv1d layer optimized for decoding"""
-    m = LinearizedConvolution(in_channels, out_channels, kernel_size, **kwargs)
-    std = math.sqrt((4 * (1.0 - dropout)) / (m.kernel_size[0] * in_channels))
-    m.weight.data.normal_(mean=0, std=std)
-    m.bias.data.zero_()
-    return m
-
-
-def ConvTBC(in_channels, out_channels, kernel_size, dropout=0.0, **kwargs):
-    """Weight-normalized Conv1d layer"""
-    from fairseq.modules import ConvTBC
-
-    m = ConvTBC(in_channels, out_channels, kernel_size, **kwargs)
-    std = math.sqrt((4 * (1.0 - dropout)) / (m.kernel_size[0] * in_channels))
-    m.weight.data.normal_(mean=0, std=std)
-    m.bias.data.zero_()
-    return m
-
-
-@register_model_architecture("fconv_self_att", "fconv_self_att")
-def base_architecture(args):
-    args.dropout = getattr(args, "dropout", 0.1)
-    args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 512)
-    args.encoder_layers = getattr(args, "encoder_layers", "[(512, 3)] * 3")
-    args.decoder_embed_dim = getattr(args, "decoder_embed_dim", 512)
-    args.decoder_layers = getattr(args, "decoder_layers", "[(512, 3)] * 8")
-    args.decoder_out_embed_dim = getattr(args, "decoder_out_embed_dim", 256)
-    args.decoder_attention = getattr(args, "decoder_attention", "True")
-    args.self_attention = getattr(args, "self_attention", "False")
-    args.encoder_attention = getattr(args, "encoder_attention", "False")
-    args.multihead_attention_nheads = getattr(args, "multihead_attention_nheads", 1)
-    args.multihead_self_attention_nheads = getattr(
-        args, "multihead_self_attention_nheads", 1
-    )
-    args.encoder_attention_nheads = getattr(args, "encoder_attention_nheads", 1)
-    args.project_input = getattr(args, "project_input", "False")
-    args.gated_attention = getattr(args, "gated_attention", "False")
-    args.downsample = getattr(args, "downsample", "False")
-    args.pretrained_checkpoint = getattr(args, "pretrained_checkpoint", "")
-    args.pretrained = getattr(args, "pretrained", "False")
-
-
-@register_model_architecture("fconv_self_att", "fconv_self_att_wp")
-def fconv_self_att_wp(args):
-    args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 256)
-    args.encoder_layers = getattr(
-        args, "encoder_layers", "[(128, 3)] * 2 + [(512,3)] * 1"
-    )
-    args.decoder_embed_dim = getattr(args, "decoder_embed_dim", 256)
-    args.decoder_layers = getattr(
-        args, "decoder_layers", "[(512, 4)] * 4 + [(768, 4)] * 2 + [(1024, 4)] * 1"
-    )
-    args.decoder_out_embed_dim = getattr(args, "decoder_out_embed_dim", 256)
-    args.self_attention = getattr(args, "self_attention", "True")
-    args.multihead_self_attention_nheads = getattr(
-        args, "multihead_self_attention_nheads", 4
-    )
-    args.project_input = getattr(args, "project_input", "True")
-    args.gated_attention = getattr(args, "gated_attention", "True")
-    args.downsample = getattr(args, "downsample", "True")
-    base_architecture(args)

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/hubert/__init__.py

@@ -1,7 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-from .hubert import *  # noqa
-from .hubert_asr import *  # noqa

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/hubert/hubert.py

@@ -1,571 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-import logging
-from dataclasses import dataclass, field
-from typing import Dict, List, Optional, Tuple
-
-import numpy as np
-import torch
-import torch.nn as nn
-from fairseq import utils
-from fairseq.data.data_utils import compute_mask_indices
-from fairseq.data.dictionary import Dictionary
-from fairseq.dataclass import ChoiceEnum, FairseqDataclass
-from fairseq.models import BaseFairseqModel, register_model
-from fairseq.models.wav2vec.wav2vec2 import (
-    EXTRACTOR_MODE_CHOICES,
-    LAYER_TYPE_CHOICES,
-    MASKING_DISTRIBUTION_CHOICES,
-    ConvFeatureExtractionModel,
-    TransformerEncoder,
-)
-from fairseq.modules import GradMultiply, LayerNorm
-from fairseq.tasks.hubert_pretraining import (
-    HubertPretrainingConfig,
-    HubertPretrainingTask,
-)
-from omegaconf import II
-
-logger = logging.getLogger(__name__)
-
-
-@dataclass
-class HubertConfig(FairseqDataclass):
-    label_rate: float = II("task.label_rate")
-
-    extractor_mode: EXTRACTOR_MODE_CHOICES = field(
-        default="default",
-        metadata={
-            "help": "mode for feature extractor. default has a single group "
-            "norm with d groups in the first conv block, whereas layer_norm "
-            "has layer norms in every block (meant to use with normalize=True)"
-        },
-    )
-    encoder_layers: int = field(
-        default=12, metadata={"help": "num encoder layers in the transformer"}
-    )
-    encoder_embed_dim: int = field(
-        default=768, metadata={"help": "encoder embedding dimension"}
-    )
-    encoder_ffn_embed_dim: int = field(
-        default=3072, metadata={"help": "encoder embedding dimension for FFN"}
-    )
-    encoder_attention_heads: int = field(
-        default=12, metadata={"help": "num encoder attention heads"}
-    )
-    activation_fn: ChoiceEnum(utils.get_available_activation_fns()) = field(
-        default="gelu", metadata={"help": "activation function to use"}
-    )
-    layer_type: LAYER_TYPE_CHOICES = field(
-        default="transformer", metadata={"help": "layer type in encoder"}
-    )
-
-    # dropouts
-    dropout: float = field(
-        default=0.1,
-        metadata={"help": "dropout probability for the transformer"},
-    )
-    attention_dropout: float = field(
-        default=0.1,
-        metadata={"help": "dropout probability for attention weights"},
-    )
-    activation_dropout: float = field(
-        default=0.0,
-        metadata={"help": "dropout probability after activation in FFN"},
-    )
-    encoder_layerdrop: float = field(
-        default=0.0,
-        metadata={"help": "probability of dropping a tarnsformer layer"},
-    )
-    dropout_input: float = field(
-        default=0.0,
-        metadata={"help": "dropout to apply to the input (after feat extr)"},
-    )
-    dropout_features: float = field(
-        default=0.0,
-        metadata={"help": "dropout to apply to the features (after feat extr)"},
-    )
-
-    final_dim: int = field(
-        default=0,
-        metadata={
-            "help": "project final representations and targets to this many "
-            "dimensions. set to encoder_embed_dim is <= 0"
-        },
-    )
-    untie_final_proj: bool = field(
-        default=False,
-        metadata={"help": "use separate projection for each target"},
-    )
-    layer_norm_first: bool = field(
-        default=False,
-        metadata={"help": "apply layernorm first in the transformer"},
-    )
-    conv_feature_layers: str = field(
-        default="[(512,10,5)] + [(512,3,2)] * 4 + [(512,2,2)] * 2",
-        metadata={
-            "help": "string describing convolutional feature extraction "
-            "layers in form of a python list that contains "
-            "[(dim, kernel_size, stride), ...]"
-        },
-    )
-    conv_bias: bool = field(
-        default=False, metadata={"help": "include bias in conv encoder"}
-    )
-    logit_temp: float = field(
-        default=0.1, metadata={"help": "temperature to divide logits by"}
-    )
-    target_glu: bool = field(
-        default=False, metadata={"help": "adds projection + glu to targets"}
-    )
-    feature_grad_mult: float = field(
-        default=1.0,
-        metadata={"help": "multiply feature extractor var grads by this"},
-    )
-
-    # masking
-    mask_length: int = field(default=10, metadata={"help": "mask length"})
-    mask_prob: float = field(
-        default=0.65,
-        metadata={"help": "probability of replacing a token with mask"},
-    )
-    mask_selection: MASKING_DISTRIBUTION_CHOICES = field(
-        default="static", metadata={"help": "how to choose mask length"}
-    )
-    mask_other: float = field(
-        default=0,
-        metadata={
-            "help": "secondary mask argument "
-            "(used for more complex distributions), "
-            "see help in compute_mask_indicesh"
-        },
-    )
-    no_mask_overlap: bool = field(
-        default=False, metadata={"help": "whether to allow masks to overlap"}
-    )
-    mask_min_space: int = field(
-        default=1,
-        metadata={"help": "min space between spans (if no overlap is enabled)"},
-    )
-
-    # channel masking
-    mask_channel_length: int = field(
-        default=10,
-        metadata={"help": "length of the mask for features (channels)"},
-    )
-    mask_channel_prob: float = field(
-        default=0.0,
-        metadata={"help": "probability of replacing a feature with 0"},
-    )
-    mask_channel_selection: MASKING_DISTRIBUTION_CHOICES = field(
-        default="static",
-        metadata={"help": "how to choose mask length for channel masking"},
-    )
-    mask_channel_other: float = field(
-        default=0,
-        metadata={
-            "help": "secondary mask argument "
-            "(used for more complex distributions), "
-            "see help in compute_mask_indicesh"
-        },
-    )
-    no_mask_channel_overlap: bool = field(
-        default=False,
-        metadata={"help": "whether to allow channel masks to overlap"},
-    )
-    mask_channel_min_space: int = field(
-        default=1,
-        metadata={"help": "min space between spans (if no overlap is enabled)"},
-    )
-
-    # positional embeddings
-    conv_pos: int = field(
-        default=128,
-        metadata={"help": "number of filters for convolutional positional embeddings"},
-    )
-    conv_pos_groups: int = field(
-        default=16,
-        metadata={"help": "number of groups for convolutional positional embedding"},
-    )
-
-    latent_temp: Tuple[float, float, float] = field(
-        default=(2, 0.5, 0.999995),
-        metadata={"help": "legacy (to be removed)"},
-    )
-
-    # loss computation
-    skip_masked: bool = field(
-        default=False,
-        metadata={"help": "skip computing losses over masked frames"},
-    )
-    skip_nomask: bool = field(
-        default=False,
-        metadata={"help": "skip computing losses over unmasked frames"},
-    )
-
-    checkpoint_activations: bool = field(
-        default=False,
-        metadata={"help": "recompute activations and save memory for extra compute"},
-    )
-
-    # FP16 optimization
-    required_seq_len_multiple: int = field(
-        default=2,
-        metadata={
-            "help": "pad the input to encoder such that the sequence length is divisible by multiple"
-        },
-    )
-
-    # Conformer
-    depthwise_conv_kernel_size: int = field(
-        default=31,
-        metadata={
-            "help": "depthwise-conv-kernel-size for convolution in conformer layer"
-        },
-    )
-    attn_type: str = field(
-        default="",
-        metadata={"help": "if espnet use ESPNET MHA"},
-    )
-    pos_enc_type: str = field(
-        default="abs",
-        metadata={"help": "Positional encoding type to use in conformer"},
-    )
-    fp16: bool = field(default=False, metadata={"help": "If fp16 is being used"})
-
-
-@register_model("hubert", dataclass=HubertConfig)
-class HubertModel(BaseFairseqModel):
-    def __init__(
-        self,
-        cfg: HubertConfig,
-        task_cfg: HubertPretrainingConfig,
-        dictionaries: List[Dictionary],
-    ) -> None:
-        super().__init__()
-        logger.info(f"HubertModel Config: {cfg}")
-
-        feature_enc_layers = eval(cfg.conv_feature_layers)  # noqa
-        self.embed = feature_enc_layers[-1][0]
-
-        self.feature_extractor = ConvFeatureExtractionModel(
-            conv_layers=feature_enc_layers,
-            dropout=0.0,
-            mode=cfg.extractor_mode,
-            conv_bias=cfg.conv_bias,
-        )
-        feature_ds_rate = np.prod([s for _, _, s in feature_enc_layers])
-        self.feat2tar_ratio = cfg.label_rate * feature_ds_rate / task_cfg.sample_rate
-
-        self.post_extract_proj = (
-            nn.Linear(self.embed, cfg.encoder_embed_dim)
-            if self.embed != cfg.encoder_embed_dim
-            else None
-        )
-
-        self.mask_prob = cfg.mask_prob
-        self.mask_selection = cfg.mask_selection
-        self.mask_other = cfg.mask_other
-        self.mask_length = cfg.mask_length
-        self.no_mask_overlap = cfg.no_mask_overlap
-        self.mask_min_space = cfg.mask_min_space
-
-        self.mask_channel_prob = cfg.mask_channel_prob
-        self.mask_channel_selection = cfg.mask_channel_selection
-        self.mask_channel_other = cfg.mask_channel_other
-        self.mask_channel_length = cfg.mask_channel_length
-        self.no_mask_channel_overlap = cfg.no_mask_channel_overlap
-        self.mask_channel_min_space = cfg.mask_channel_min_space
-
-        self.dropout_input = nn.Dropout(cfg.dropout_input)
-        self.dropout_features = nn.Dropout(cfg.dropout_features)
-
-        self.feature_grad_mult = cfg.feature_grad_mult
-        self.logit_temp = cfg.logit_temp
-        self.skip_masked = cfg.skip_masked
-        self.skip_nomask = cfg.skip_nomask
-
-        final_dim = cfg.final_dim if cfg.final_dim > 0 else cfg.encoder_embed_dim
-
-        self.mask_emb = nn.Parameter(
-            torch.FloatTensor(cfg.encoder_embed_dim).uniform_()
-        )
-
-        self.encoder = TransformerEncoder(cfg)
-        self.layer_norm = LayerNorm(self.embed)
-
-        self.target_glu = None
-        if cfg.target_glu:
-            self.target_glu = nn.Sequential(
-                nn.Linear(final_dim, final_dim * 2), nn.GLU()
-            )
-
-        self.untie_final_proj = cfg.untie_final_proj
-        if self.untie_final_proj:
-            self.final_proj = nn.Linear(
-                cfg.encoder_embed_dim, final_dim * len(dictionaries)
-            )
-        else:
-            self.final_proj = nn.Linear(cfg.encoder_embed_dim, final_dim)
-
-        # modules below are not needed during fine-tuning
-        if any([d is None for d in dictionaries]):
-            logger.info("cannot find dictionary. assume will be used for fine-tuning")
-        else:
-            self.num_classes = [len(d) for d in dictionaries]
-            self.label_embs_concat = nn.Parameter(
-                torch.FloatTensor(sum(self.num_classes), final_dim)
-            )
-            nn.init.uniform_(self.label_embs_concat)
-
-    def upgrade_state_dict_named(self, state_dict, name):
-        """Upgrade a (possibly old) state dict for new versions of fairseq."""
-
-        super().upgrade_state_dict_named(state_dict, name)
-        return state_dict
-
-    @classmethod
-    def build_model(cls, cfg: HubertConfig, task: HubertPretrainingTask):
-        """Build a new model instance."""
-
-        model = HubertModel(cfg, task.cfg, task.dictionaries)
-        return model
-
-    def apply_mask(self, x, padding_mask, target_list):
-        B, T, C = x.shape
-        if self.mask_prob > 0:
-            mask_indices = compute_mask_indices(
-                (B, T),
-                padding_mask,
-                self.mask_prob,
-                self.mask_length,
-                self.mask_selection,
-                self.mask_other,
-                min_masks=2,
-                no_overlap=self.no_mask_overlap,
-                min_space=self.mask_min_space,
-            )
-            mask_indices = torch.from_numpy(mask_indices).to(x.device)
-            x[mask_indices] = self.mask_emb
-        else:
-            mask_indices = None
-
-        if self.mask_channel_prob > 0:
-            mask_channel_indices = compute_mask_indices(
-                (B, C),
-                None,
-                self.mask_channel_prob,
-                self.mask_channel_length,
-                self.mask_channel_selection,
-                self.mask_channel_other,
-                no_overlap=self.no_mask_channel_overlap,
-                min_space=self.mask_channel_min_space,
-            )
-            mask_channel_indices = (
-                torch.from_numpy(mask_channel_indices)
-                .to(x.device)
-                .unsqueeze(1)
-                .expand(-1, T, -1)
-            )
-            x[mask_channel_indices] = 0
-
-        return x, mask_indices
-
-    def compute_nce(self, x, pos, negs):
-        neg_is_pos = (pos == negs).all(-1)
-        pos = pos.unsqueeze(0)
-        targets = torch.cat([pos, negs], dim=0)
-
-        logits = torch.cosine_similarity(x.float(), targets.float(), dim=-1).type_as(x)
-        logits /= self.logit_temp
-        if neg_is_pos.any():
-            logits[1:][neg_is_pos] = float("-inf")
-        logits = logits.transpose(0, 1)  # (num_x, num_cls+1)
-        return logits
-
-    def forward_features(self, source: torch.Tensor) -> torch.Tensor:
-        if self.feature_grad_mult > 0:
-            features = self.feature_extractor(source)
-            if self.feature_grad_mult != 1.0:
-                features = GradMultiply.apply(features, self.feature_grad_mult)
-        else:
-            with torch.no_grad():
-                features = self.feature_extractor(source)
-        return features
-
-    def forward_targets(
-        self,
-        features: torch.Tensor,
-        target_list: List[torch.Tensor],
-    ) -> Tuple[torch.Tensor, torch.Tensor]:
-        # Trim features to ensure labels exist and then get aligned labels
-        feat_tsz = features.size(2)
-        targ_tsz = min([t.size(1) for t in target_list])
-        if self.feat2tar_ratio * feat_tsz > targ_tsz:
-            feat_tsz = int(targ_tsz / self.feat2tar_ratio)
-            features = features[..., :feat_tsz]
-        target_inds = torch.arange(feat_tsz).float() * self.feat2tar_ratio
-        target_list = [t[:, target_inds.long()] for t in target_list]
-        return features, target_list
-
-    def forward_padding_mask(
-        self,
-        features: torch.Tensor,
-        padding_mask: torch.Tensor,
-    ) -> torch.Tensor:
-        extra = padding_mask.size(1) % features.size(1)
-        if extra > 0:
-            padding_mask = padding_mask[:, :-extra]
-        padding_mask = padding_mask.view(padding_mask.size(0), features.size(1), -1)
-        padding_mask = padding_mask.all(-1)
-        return padding_mask
-
-    def forward(
-        self,
-        source: torch.Tensor,
-        target_list: Optional[List[torch.Tensor]] = None,
-        padding_mask: Optional[torch.Tensor] = None,
-        mask: bool = True,
-        features_only: bool = False,
-        output_layer: Optional[int] = None,
-    ) -> Dict[str, torch.Tensor]:
-        """output layer is 1-based"""
-        features = self.forward_features(source)
-        if target_list is not None:
-            features, target_list = self.forward_targets(features, target_list)
-
-        features_pen = features.float().pow(2).mean()
-
-        features = features.transpose(1, 2)
-        features = self.layer_norm(features)
-        unmasked_features = features.clone()
-
-        if padding_mask is not None:
-            padding_mask = self.forward_padding_mask(features, padding_mask)
-
-        if self.post_extract_proj is not None:
-            features = self.post_extract_proj(features)
-
-        features = self.dropout_input(features)
-        unmasked_features = self.dropout_features(unmasked_features)
-
-        if mask:
-            x, mask_indices = self.apply_mask(features, padding_mask, target_list)
-        else:
-            x = features
-            mask_indices = None
-
-        # feature: (B, T, D), float
-        # target: (B, T), long
-        # x: (B, T, D), float
-        # padding_mask: (B, T), bool
-        # mask_indices: (B, T), bool
-        x, layer_results = self.encoder(
-            x,
-            padding_mask=padding_mask,
-            layer=None if output_layer is None else output_layer - 1,
-        )
-
-        if features_only:
-            return {"x": x, "padding_mask": padding_mask, "features": features,"layer_results":layer_results}
-
-        def compute_pred(proj_x, target, label_embs):
-            # compute logits for the i-th label set
-            y = torch.index_select(label_embs, 0, target.long())
-            negs = label_embs.unsqueeze(1).expand(-1, proj_x.size(0), -1)
-            if self.target_glu:
-                y = self.target_glu(y)
-                negs = self.target_glu(negs)
-            # proj_x: (S, D)
-            # y: (S, D)
-            # negs: (Neg, S, D)
-            return self.compute_nce(proj_x, y, negs)
-
-        label_embs_list = self.label_embs_concat.split(self.num_classes, 0)
-
-        if not self.skip_masked:
-            masked_indices = torch.logical_and(~padding_mask, mask_indices)
-            proj_x_m = self.final_proj(x[masked_indices])
-            if self.untie_final_proj:
-                proj_x_m_list = proj_x_m.chunk(len(target_list), dim=-1)
-            else:
-                proj_x_m_list = [proj_x_m for _ in range(len(target_list))]
-            logit_m_list = [
-                compute_pred(proj_x_m, t[masked_indices], label_embs_list[i])
-                for i, (proj_x_m, t) in enumerate(zip(proj_x_m_list, target_list))
-            ]
-        else:
-            logit_m_list = [None for _ in target_list]
-
-        if not self.skip_nomask:
-            nomask_indices = torch.logical_and(~padding_mask, ~mask_indices)
-            proj_x_u = self.final_proj(x[nomask_indices])
-            if self.untie_final_proj:
-                proj_x_u_list = proj_x_u.chunk(len(target_list), dim=-1)
-            else:
-                proj_x_u_list = [proj_x_u for _ in range(len(target_list))]
-
-            logit_u_list = [
-                compute_pred(proj_x_u, t[nomask_indices], label_embs_list[i])
-                for i, (proj_x_u, t) in enumerate(zip(proj_x_u_list, target_list))
-            ]
-        else:
-            logit_u_list = [None for _ in target_list]
-
-        result = {
-            "logit_m_list": logit_m_list,
-            "logit_u_list": logit_u_list,
-            "padding_mask": padding_mask,
-            "features_pen": features_pen,
-        }
-        return result
-
-    def extract_features(
-        self,
-        source: torch.Tensor,
-        padding_mask: Optional[torch.Tensor] = None,
-        mask: bool = False,
-        ret_conv: bool = False,
-        output_layer: Optional[int] = None,
-    ) -> Tuple[torch.Tensor, torch.Tensor]:
-        res = self.forward(
-            source,
-            padding_mask=padding_mask,
-            mask=mask,
-            features_only=True,
-            output_layer=output_layer,
-        )
-        feature = res["features"] if ret_conv else res["x"]
-        layer_results = res["layer_results"]
-        avg_layer_results = sum([result[0] for result in layer_results]) / len(layer_results)  
-        return feature, res["padding_mask"], avg_layer_results
-
-    def get_logits(self, net_output, is_masked=True):
-        if is_masked:
-            logits_list = net_output["logit_m_list"]
-        else:
-            logits_list = net_output["logit_u_list"]
-        logits_list = [x.float() for x in logits_list if x is not None]
-        return logits_list
-
-    def get_targets(self, net_output, is_masked=True):
-        logits_list = self.get_logits(net_output, is_masked)
-        targets_list = [x.new_zeros(x.size(0), dtype=torch.long) for x in logits_list]
-        return targets_list
-
-    def get_extra_losses(self, net_output):
-        extra_losses = []
-        names = []
-
-        if "features_pen" in net_output:
-            extra_losses.append(net_output["features_pen"])
-            names.append("features_pen")
-
-        return extra_losses, names
-
-    def remove_pretraining_modules(self):
-        self.target_glu = None
-        self.final_proj = None

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/hubert/hubert_asr.py

@@ -1,363 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-import contextlib
-from argparse import Namespace
-from dataclasses import dataclass, field
-from typing import Any
-
-import torch
-import torch.nn as nn
-from fairseq import checkpoint_utils, tasks, utils
-from fairseq.dataclass import FairseqDataclass
-from fairseq.dataclass.utils import convert_namespace_to_omegaconf
-from fairseq.models import BaseFairseqModel, FairseqEncoder, register_model
-from fairseq.models.hubert.hubert import MASKING_DISTRIBUTION_CHOICES
-from fairseq.tasks import FairseqTask
-from omegaconf import II, MISSING
-
-
-@dataclass
-class HubertAsrConfig(FairseqDataclass):
-    w2v_path: str = field(default=MISSING, metadata={"help": "path to hubert model"})
-    no_pretrained_weights: bool = field(
-        default=False,
-        metadata={"help": "if true, does not load pretrained weights"},
-    )
-    dropout_input: float = field(
-        default=0.0,
-        metadata={"help": "dropout to apply to the input (after feat extr)"},
-    )
-    final_dropout: float = field(
-        default=0.0,
-        metadata={"help": "dropout after transformer and before final projection"},
-    )
-    dropout: float = field(
-        default=0.0,
-        metadata={"help": "dropout probability inside hubert model"},
-    )
-    attention_dropout: float = field(
-        default=0.0,
-        metadata={
-            "help": "dropout probability for attention weights " "inside hubert model"
-        },
-    )
-    activation_dropout: float = field(
-        default=0.0,
-        metadata={
-            "help": "dropout probability after activation in FFN " "inside hubert model"
-        },
-    )
-
-    # masking
-    apply_mask: bool = field(
-        default=False, metadata={"help": "apply masking during fine-tuning"}
-    )
-    mask_length: int = field(
-        default=10, metadata={"help": "repeat the mask indices multiple times"}
-    )
-    mask_prob: float = field(
-        default=0.5,
-        metadata={
-            "help": "probability of replacing a token with mask "
-            "(normalized by length)"
-        },
-    )
-    mask_selection: MASKING_DISTRIBUTION_CHOICES = field(
-        default="static", metadata={"help": "how to choose masks"}
-    )
-    mask_other: float = field(
-        default=0,
-        metadata={
-            "help": "secondary mask argument "
-            "(used for more complex distributions), "
-            "see help in compute_mask_indices"
-        },
-    )
-    no_mask_overlap: bool = field(
-        default=False, metadata={"help": "whether to allow masks to overlap"}
-    )
-
-    # channel masking
-    mask_channel_length: int = field(
-        default=10,
-        metadata={"help": "length of the mask for features (channels)"},
-    )
-    mask_channel_prob: float = field(
-        default=0.0,
-        metadata={"help": "probability of replacing a feature with 0"},
-    )
-    mask_channel_selection: MASKING_DISTRIBUTION_CHOICES = field(
-        default="static",
-        metadata={"help": "how to choose mask length for channel masking"},
-    )
-    mask_channel_other: float = field(
-        default=0,
-        metadata={
-            "help": "secondary mask argument "
-            "(used for more complex distributions), "
-            "see help in compute_mask_indices"
-        },
-    )
-    no_mask_channel_overlap: bool = field(
-        default=False,
-        metadata={"help": "whether to allow channel masks to overlap"},
-    )
-    freeze_finetune_updates: int = field(
-        default=0,
-        metadata={"help": "dont finetune hubert for this many updates"},
-    )
-    feature_grad_mult: float = field(
-        default=0.0,
-        metadata={"help": "reset feature grad mult in hubert to this"},
-    )
-    layerdrop: float = field(
-        default=0.0,
-        metadata={"help": "probability of dropping a layer in hubert"},
-    )
-    normalize: bool = II("task.normalize")
-    data: str = II("task.data")
-
-    # this holds the loaded hubert args
-    w2v_args: Any = None
-
-
-@dataclass
-class HubertCtcConfig(HubertAsrConfig):
-    pass
-
-
-@register_model("hubert_ctc", dataclass=HubertCtcConfig)
-class HubertCtc(BaseFairseqModel):
-    def __init__(self, cfg: HubertCtcConfig, w2v_encoder: BaseFairseqModel):
-        super().__init__()
-        self.cfg = cfg
-        self.w2v_encoder = w2v_encoder
-
-    def upgrade_state_dict_named(self, state_dict, name):
-        super().upgrade_state_dict_named(state_dict, name)
-        return state_dict
-
-    @classmethod
-    def build_model(cls, cfg: HubertCtcConfig, task: FairseqTask):
-        """Build a new model instance."""
-        w2v_encoder = HubertEncoder(cfg, task)
-        return cls(cfg, w2v_encoder)
-
-    def get_normalized_probs(self, net_output, log_probs):
-        """Get normalized probabilities (or log probs) from a net's output."""
-
-        logits = net_output["encoder_out"]
-        if log_probs:
-            return utils.log_softmax(logits.float(), dim=-1)
-        else:
-            return utils.softmax(logits.float(), dim=-1)
-
-    def get_logits(self, net_output):
-        logits = net_output["encoder_out"]
-        padding = net_output["encoder_padding_mask"]
-        if padding is not None and padding.any():
-            padding = padding.T
-            logits[padding][..., 0] = 0
-            logits[padding][..., 1:] = float("-inf")
-
-        return logits
-
-    def forward(self, **kwargs):
-        x = self.w2v_encoder(**kwargs)
-        return x
-
-
-@dataclass
-class HubertSeq2SeqConfig(HubertAsrConfig):
-    decoder_embed_dim: int = field(
-        default=768, metadata={"help": "decoder embedding dimension"}
-    )
-    decoder_ffn_embed_dim: int = field(
-        default=3072, metadata={"help": "decoder embedding dimension for FFN"}
-    )
-    decoder_layers: int = field(default=6, metadata={"help": "num of decoder layers"})
-    decoder_layerdrop: float = field(
-        default=0.0, metadata={"help": "decoder layerdrop chance"}
-    )
-    decoder_attention_heads: int = field(
-        default=4, metadata={"help": "num decoder attention heads"}
-    )
-    decoder_learned_pos: bool = field(
-        default=False,
-        metadata={"help": "use learned positional embeddings in the decoder"},
-    )
-    decoder_normalize_before: bool = field(
-        default=False,
-        metadata={"help": "apply layernorm before each decoder block"},
-    )
-    no_token_positional_embeddings: bool = field(
-        default=False,
-        metadata={
-            "help": "if set, disables positional embeddings " "(outside self attention)"
-        },
-    )
-    decoder_dropout: float = field(
-        default=0.0, metadata={"help": "dropout probability in the decoder"}
-    )
-    decoder_attention_dropout: float = field(
-        default=0.0,
-        metadata={
-            "help": "dropout probability for attention weights " "inside the decoder"
-        },
-    )
-    decoder_activation_dropout: float = field(
-        default=0.0,
-        metadata={
-            "help": "dropout probability after activation in FFN " "inside the decoder"
-        },
-    )
-    max_target_positions: int = field(
-        default=2048, metadata={"help": "max target positions"}
-    )
-    share_decoder_input_output_embed: bool = field(
-        default=False,
-        metadata={"help": "share decoder input and output embeddings"},
-    )
-
-
-class HubertEncoder(FairseqEncoder):
-    def __init__(self, cfg: HubertAsrConfig, task):
-        self.apply_mask = cfg.apply_mask
-
-        arg_overrides = {
-            "dropout": cfg.dropout,
-            "activation_dropout": cfg.activation_dropout,
-            "dropout_input": cfg.dropout_input,
-            "attention_dropout": cfg.attention_dropout,
-            "mask_length": cfg.mask_length,
-            "mask_prob": cfg.mask_prob,
-            "mask_selection": cfg.mask_selection,
-            "mask_other": cfg.mask_other,
-            "no_mask_overlap": cfg.no_mask_overlap,
-            "mask_channel_length": cfg.mask_channel_length,
-            "mask_channel_prob": cfg.mask_channel_prob,
-            "mask_channel_selection": cfg.mask_channel_selection,
-            "mask_channel_other": cfg.mask_channel_other,
-            "no_mask_channel_overlap": cfg.no_mask_channel_overlap,
-            "encoder_layerdrop": cfg.layerdrop,
-            "feature_grad_mult": cfg.feature_grad_mult,
-        }
-
-        if cfg.w2v_args is None:
-            state = checkpoint_utils.load_checkpoint_to_cpu(cfg.w2v_path, arg_overrides)
-            w2v_args = state.get("cfg", None)
-            if w2v_args is None:
-                w2v_args = convert_namespace_to_omegaconf(state["args"])
-            cfg.w2v_args = w2v_args
-        else:
-            state = None
-            w2v_args = cfg.w2v_args
-            if isinstance(w2v_args, Namespace):
-                cfg.w2v_args = w2v_args = convert_namespace_to_omegaconf(w2v_args)
-
-        assert cfg.normalize == w2v_args.task.normalize, (
-            "Fine-tuning works best when data normalization is the same. "
-            "Please check that --normalize is set or unset for "
-            "both pre-training and here"
-        )
-
-        w2v_args.task.data = cfg.data
-        pretrain_task = tasks.setup_task(w2v_args.task)
-        if state is not None and "task_state" in state:
-            # This will load the stored "dictionaries" object
-            pretrain_task.load_state_dict(state["task_state"])
-        else:
-            pretrain_task.load_state_dict(task.state_dict())
-
-        model = pretrain_task.build_model(w2v_args.model, from_checkpoint=True)
-        if state is not None and not cfg.no_pretrained_weights:
-            # set strict=False because we omit some modules
-            model.load_state_dict(state["model"], strict=False)
-
-        model.remove_pretraining_modules()
-
-        super().__init__(pretrain_task.source_dictionary)
-
-        d = w2v_args.model.encoder_embed_dim
-
-        self.w2v_model = model
-
-        self.final_dropout = nn.Dropout(cfg.final_dropout)
-        self.freeze_finetune_updates = cfg.freeze_finetune_updates
-        self.num_updates = 0
-
-        if task.target_dictionary is not None:
-            self.proj = Linear(d, len(task.target_dictionary))
-        elif getattr(cfg, "decoder_embed_dim", d) != d:
-            self.proj = Linear(d, cfg.decoder_embed_dim)
-        else:
-            self.proj = None
-
-    def set_num_updates(self, num_updates):
-        """Set the number of parameters updates."""
-        super().set_num_updates(num_updates)
-        self.num_updates = num_updates
-
-    def forward(self, source, padding_mask, tbc=True, **kwargs):
-
-        w2v_args = {
-            "source": source,
-            "padding_mask": padding_mask,
-            "mask": self.apply_mask and self.training,
-        }
-
-        ft = self.freeze_finetune_updates <= self.num_updates
-
-        with torch.no_grad() if not ft else contextlib.ExitStack():
-            x, padding_mask = self.w2v_model.extract_features(**w2v_args)
-
-            if tbc:
-                # B x T x C -> T x B x C
-                x = x.transpose(0, 1)
-
-        x = self.final_dropout(x)
-
-        if self.proj:
-            x = self.proj(x)
-
-        return {
-            "encoder_out": x,  # T x B x C
-            "encoder_padding_mask": padding_mask,  # B x T
-            "padding_mask": padding_mask,
-        }
-
-    def reorder_encoder_out(self, encoder_out, new_order):
-        if encoder_out["encoder_out"] is not None:
-            encoder_out["encoder_out"] = encoder_out["encoder_out"].index_select(
-                1, new_order
-            )
-        if encoder_out["encoder_padding_mask"] is not None:
-            encoder_out["encoder_padding_mask"] = encoder_out[
-                "encoder_padding_mask"
-            ].index_select(0, new_order)
-        return encoder_out
-
-    def max_positions(self):
-        """Maximum input length supported by the encoder."""
-        return None
-
-    def upgrade_state_dict_named(self, state_dict, name):
-        return state_dict
-
-
-def Embedding(num_embeddings, embedding_dim, padding_idx):
-    m = nn.Embedding(num_embeddings, embedding_dim, padding_idx=padding_idx)
-    nn.init.normal_(m.weight, mean=0, std=embedding_dim**-0.5)
-    nn.init.constant_(m.weight[padding_idx], 0)
-    return m
-
-
-def Linear(in_features, out_features, bias=True):
-    m = nn.Linear(in_features, out_features, bias)
-    nn.init.xavier_uniform_(m.weight)
-    if bias:
-        nn.init.constant_(m.bias, 0.0)
-    return m

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/huggingface/__init__.py

@@ -1,19 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-import importlib
-import os
-
-# automatically import any Python files in the models/huggingface/ directory
-models_dir = os.path.dirname(__file__)
-for file in os.listdir(models_dir):
-    path = os.path.join(models_dir, file)
-    if (
-        not file.startswith("_")
-        and not file.startswith(".")
-        and (file.endswith(".py") or os.path.isdir(path))
-    ):
-        model_name = file[: file.find(".py")] if file.endswith(".py") else file
-        module = importlib.import_module("fairseq.models.huggingface." + model_name)

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/huggingface/hf_gpt2.py

@@ -1,167 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-import logging
-import os
-import sys
-from typing import Dict, List, Optional
-
-import torch
-from fairseq.models import (
-    FairseqIncrementalDecoder,
-    FairseqLanguageModel,
-    register_model,
-    register_model_architecture,
-)
-
-logger = logging.getLogger(__name__)
-
-
-DEFAULT_MAX_TARGET_POSITIONS = 1024
-
-
-@register_model("hf_gpt2")
-class HuggingFaceGPT2LanguageModel(FairseqLanguageModel):
-    def __init__(self, decoder):
-        super().__init__(decoder)
-
-    @staticmethod
-    def add_args(parser):
-        """Add model-specific arguments to the parser."""
-        # fmt: off
-        parser.add_argument('--embed-dim', type=int, metavar='N',
-                            help='embedding dimension')
-        parser.add_argument('--num-attention-heads', type=int, metavar='N',
-                            help='num attention heads')
-        parser.add_argument('--num-layers', type=int, metavar='N',
-                            help='num layers')
-        parser.add_argument('--dropout', type=float, metavar='D',
-                            help='dropout probability for all fully connected layers '
-                                 'in the embeddings, encoder, and pooler')
-        parser.add_argument('--attention-dropout', type=float, metavar='D',
-                            help='dropout probability for attention weights')
-        # fmt: on
-
-    @classmethod
-    def build_model(cls, args, task):
-        """Build a new model instance."""
-        default_architecture(args)
-        return cls(HuggingFaceGPT2Decoder(args, task))
-
-
-class HuggingFaceGPT2Decoder(FairseqIncrementalDecoder):
-    def __init__(self, args, task):
-        try:
-            from transformers import GPT2Config, GPT2LMHeadModel
-        except ImportError:
-            raise ImportError(
-                "\n\nPlease install huggingface/transformers with:"
-                "\n\n  pip install transformers"
-            )
-
-        super().__init__(task.target_dictionary)
-
-        config = GPT2Config(
-            vocab_size=len(task.target_dictionary),
-            n_positions=args.max_target_positions + 1,
-            n_ctx=args.max_target_positions,
-            n_embd=args.embed_dim,
-            n_layer=args.num_layers,
-            n_head=args.num_attention_heads,
-            resid_pdrop=args.dropout,
-            embd_pdrop=args.dropout,
-            attn_pdrop=args.attention_dropout,
-            layer_norm_epsilon=1e-6,
-        )
-        self.model = GPT2LMHeadModel(config)
-
-        # set zero embedding for padding symbol
-        self.pad_idx = task.target_dictionary.pad()
-        self.model.transformer.wte.weight.data[self.pad_idx].zero_()
-        self.model.transformer.wpe.weight.data[0].zero_()
-
-    def forward(
-        self,
-        prev_output_tokens,
-        src_lengths=None,
-        incremental_state: Optional[Dict[str, List[torch.Tensor]]] = None,
-        encoder_out=None,
-    ):
-        features = self.extract_features(prev_output_tokens, incremental_state)
-        lm_logits = self.model.lm_head(features)
-        return (lm_logits,)
-
-    def extract_features(
-        self,
-        prev_output_tokens,
-        incremental_state: Optional[Dict[str, List[torch.Tensor]]] = None,
-    ):
-        if incremental_state:
-            past = self.get_incremental_state("past")
-        else:
-            past = None
-
-        # don't attend to padding symbols
-        attention_mask = prev_output_tokens.ne(self.pad_idx).int()
-
-        # set position ids to exclude padding symbols
-        position_ids = attention_mask * (
-            torch.arange(1, 1 + prev_output_tokens.size(1))
-            .to(prev_output_tokens)
-            .repeat(prev_output_tokens.size(0), 1)
-        )
-
-        outputs = self.model.transformer(
-            input_ids=prev_output_tokens,
-            past=past,
-            attention_mask=attention_mask,
-            position_ids=position_ids,
-        )
-        last_hidden_states = outputs[0]
-
-        if incremental_state:
-            self.set_incremental_state(incremental_state, "past", outputs[1])
-
-        return last_hidden_states
-
-    def max_positions(self):
-        return self.model.config.n_positions - 1
-
-
-@register_model_architecture("hf_gpt2", "hf_gpt2")
-def default_architecture(args):
-    if getattr(args, "max_target_positions", None) is None:
-        args.max_target_positions = getattr(
-            args, "tokens_per_sample", DEFAULT_MAX_TARGET_POSITIONS
-        )
-    args.embed_dim = getattr(args, "embed_dim", 768)
-    args.num_attention_heads = getattr(args, "num_attention_heads", 12)
-    args.num_layers = getattr(args, "num_layers", 12)
-    args.dropout = getattr(args, "dropout", 0.1)
-    args.attention_dropout = getattr(args, "attention_dropout", 0.1)
-
-
-@register_model_architecture("hf_gpt2", "hf_gpt2_medium")
-def hf_gpt2_medium(args):
-    args.embed_dim = getattr(args, "embed_dim", 1024)
-    args.num_attention_heads = getattr(args, "num_attention_heads", 16)
-    args.num_layers = getattr(args, "num_layers", 24)
-    default_architecture(args)
-
-
-@register_model_architecture("hf_gpt2", "hf_gpt2_large")
-def hf_gpt2_large(args):
-    args.embed_dim = getattr(args, "embed_dim", 1280)
-    args.num_attention_heads = getattr(args, "num_attention_heads", 20)
-    args.num_layers = getattr(args, "num_layers", 36)
-    default_architecture(args)
-
-
-@register_model_architecture("hf_gpt2", "hf_gpt2_xl")
-def hf_gpt2_xl(args):
-    args.embed_dim = getattr(args, "embed_dim", 1600)
-    args.num_attention_heads = getattr(args, "num_attention_heads", 25)
-    args.num_layers = getattr(args, "num_layers", 48)
-    default_architecture(args)

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/lightconv.py

@@ -1,1019 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-import math
-
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-from fairseq import utils
-from fairseq.models import (
-    FairseqEncoder,
-    FairseqEncoderDecoderModel,
-    FairseqIncrementalDecoder,
-    register_model,
-    register_model_architecture,
-)
-from fairseq.modules import (
-    AdaptiveSoftmax,
-    DynamicConv,
-    FairseqDropout,
-    LayerNorm,
-    LightweightConv,
-    MultiheadAttention,
-    PositionalEmbedding,
-)
-from fairseq.utils import safe_hasattr
-
-
-@register_model("lightconv")
-class LightConvModel(FairseqEncoderDecoderModel):
-    """
-    LightConv and DynamicConv model from `"Pay Less Attention with Lightweight and Dynamic Convolutions" (Wu, et al, 2019)
-    <https://openreview.net/pdf?id=SkVhlh09tX>`_.
-    To use LightConv please set ``--encoder-conv-type lightweight --decoder-conv-type lightweight``
-    To use DynamicConv please set ``--encoder-conv-type dynamic --decoder-conv-type dynamic``
-
-    Args:
-        encoder (LightConvEncoder): the encoder
-        decoder (LightConvDecoder): the decoder
-
-    The LightConv model provides the following named architectures and
-    command-line arguments:
-
-    .. argparse::
-        :ref: fairseq.models.lightconv_parser
-        :prog:
-    """
-
-    @classmethod
-    def hub_models(cls):
-        # fmt: off
-
-        def moses_subword(path):
-            return {
-                'path': path,
-                'tokenizer': 'moses',
-                'bpe': 'subword_nmt',
-            }
-
-        return {
-            'lightconv.no_glu.iwslt14.de-en': moses_subword('https://dl.fbaipublicfiles.com/fairseq/models/dynamicconv/iwslt14.de-en.lightconv.tar.gz'),
-            'dynamicconv.no_glu.iwslt14.de-en': moses_subword('https://dl.fbaipublicfiles.com/fairseq/models/dynamicconv/iwslt14.de-en.dynamicconv.tar.gz'),
-            'lightconv.no_glu.wmt16.en-de': moses_subword('https://dl.fbaipublicfiles.com/fairseq/models/dynamicconv/wmt16.en-de.joined-dict.lightconv.tar.gz'),
-            'dynamicconv.no_glu.wmt16.en-de': moses_subword('https://dl.fbaipublicfiles.com/fairseq/models/dynamicconv/wmt16.en-de.joined-dict.dynamicconv.tar.gz'),
-            'lightconv.glu.wmt16.en-de': moses_subword('https://dl.fbaipublicfiles.com/fairseq/models/dynamicconv/wmt16.en-de.joined-dict.lightconv-glu.tar.gz'),
-            'dynamicconv.glu.wmt16.en-de': moses_subword('https://dl.fbaipublicfiles.com/fairseq/models/dynamicconv/wmt16.en-de.joined-dict.dynamicconv-glu.tar.gz'),
-            'lightconv.glu.wmt17.en-de': moses_subword('https://dl.fbaipublicfiles.com/fairseq/models/dynamicconv/wmt16.en-de.joined-dict.lightconv-glu.tar.gz'),
-            'dynamicconv.glu.wmt17.en-de': moses_subword('https://dl.fbaipublicfiles.com/fairseq/models/dynamicconv/wmt16.en-de.joined-dict.dynamicconv-glu.tar.gz'),
-            'lightconv.glu.wmt14.en-fr': moses_subword('https://dl.fbaipublicfiles.com/fairseq/models/dynamicconv/wmt14.en-fr.joined-dict.lightconv-glu.tar.gz'),
-            'dynamicconv.glu.wmt14.en-fr': moses_subword('https://dl.fbaipublicfiles.com/fairseq/models/dynamicconv/wmt14.en-fr.joined-dict.dynamicconv-glu.tar.gz'),
-            'lightconv.glu.wmt17.zh-en': moses_subword('https://dl.fbaipublicfiles.com/fairseq/models/dynamicconv/wmt17.zh-en.lightconv-glu.tar.gz'),
-            'dynamicconv.glu.wmt17.zh-en': moses_subword('https://dl.fbaipublicfiles.com/fairseq/models/dynamicconv/wmt17.zh-en.dynamicconv-glu.tar.gz'),
-        }
-        # fmt: on
-
-    def __init__(self, encoder, decoder):
-        super().__init__(encoder, decoder)
-
-    @staticmethod
-    def add_args(parser):
-        """Add model-specific arguments to the parser."""
-        parser.add_argument(
-            "--dropout", type=float, metavar="D", help="dropout probability"
-        )
-        parser.add_argument(
-            "--attention-dropout",
-            type=float,
-            metavar="D",
-            help="dropout probability for attention weights",
-        )
-        parser.add_argument(
-            "--relu-dropout",
-            type=float,
-            metavar="D",
-            help="dropout probability after ReLU in FFN",
-        )
-        parser.add_argument(
-            "--input-dropout",
-            type=float,
-            metavar="D",
-            help="dropout probability of the inputs",
-        )
-        parser.add_argument(
-            "--encoder-embed-path",
-            type=str,
-            metavar="STR",
-            help="path to pre-trained encoder embedding",
-        )
-        parser.add_argument(
-            "--encoder-embed-dim",
-            type=int,
-            metavar="N",
-            help="encoder embedding dimension",
-        )
-        parser.add_argument(
-            "--encoder-conv-dim",
-            type=int,
-            metavar="N",
-            help="encoder embedding dimension",
-        )
-        parser.add_argument(
-            "--encoder-ffn-embed-dim",
-            type=int,
-            metavar="N",
-            help="encoder embedding dimension for FFN",
-        )
-        parser.add_argument(
-            "--encoder-layers", type=int, metavar="N", help="num encoder layers"
-        )
-        parser.add_argument(
-            "--encoder-attention-heads",
-            type=int,
-            metavar="N",
-            help="num encoder attention heads or LightConv/DynamicConv heads",
-        )
-        parser.add_argument(
-            "--encoder-normalize-before",
-            action="store_true",
-            help="apply layernorm before each encoder block",
-        )
-        parser.add_argument(
-            "--encoder-learned-pos",
-            action="store_true",
-            help="use learned positional embeddings in the encoder",
-        )
-        parser.add_argument(
-            "--decoder-embed-path",
-            type=str,
-            metavar="STR",
-            help="path to pre-trained decoder embedding",
-        )
-        parser.add_argument(
-            "--decoder-embed-dim",
-            type=int,
-            metavar="N",
-            help="decoder embedding dimension",
-        )
-        parser.add_argument(
-            "--decoder-conv-dim",
-            type=int,
-            metavar="N",
-            help="decoder embedding dimension",
-        )
-        parser.add_argument(
-            "--decoder-ffn-embed-dim",
-            type=int,
-            metavar="N",
-            help="decoder embedding dimension for FFN",
-        )
-        parser.add_argument(
-            "--decoder-layers", type=int, metavar="N", help="num decoder layers"
-        )
-        parser.add_argument(
-            "--decoder-attention-heads",
-            type=int,
-            metavar="N",
-            help="num decoder attention heads or LightConv/DynamicConv heads",
-        )
-        parser.add_argument(
-            "--decoder-learned-pos",
-            action="store_true",
-            help="use learned positional embeddings in the decoder",
-        )
-        parser.add_argument(
-            "--decoder-normalize-before",
-            action="store_true",
-            help="apply layernorm before each decoder block",
-        )
-        parser.add_argument(
-            "--share-decoder-input-output-embed",
-            action="store_true",
-            help="share decoder input and output embeddings",
-        )
-        parser.add_argument(
-            "--share-all-embeddings",
-            action="store_true",
-            help="share encoder, decoder and output embeddings"
-            " (requires shared dictionary and embed dim)",
-        )
-        parser.add_argument(
-            "--adaptive-softmax-cutoff",
-            metavar="EXPR",
-            help="comma separated list of adaptive softmax cutoff points. "
-            "Must be used with adaptive_loss criterion",
-        ),
-        parser.add_argument(
-            "--adaptive-softmax-dropout",
-            type=float,
-            metavar="D",
-            help="sets adaptive softmax dropout for the tail projections",
-        )
-
-        """LightConv and DynamicConv arguments"""
-        parser.add_argument(
-            "--encoder-kernel-size-list",
-            type=lambda x: utils.eval_str_list(x, int),
-            help='list of kernel size (default: "[3,7,15,31,31,31,31]")',
-        )
-        parser.add_argument(
-            "--decoder-kernel-size-list",
-            type=lambda x: utils.eval_str_list(x, int),
-            help='list of kernel size (default: "[3,7,15,31,31,31]")',
-        )
-        parser.add_argument(
-            "--encoder-glu", type=utils.eval_bool, help="glu after in proj"
-        )
-        parser.add_argument(
-            "--decoder-glu", type=utils.eval_bool, help="glu after in proj"
-        )
-        parser.add_argument(
-            "--encoder-conv-type",
-            default="dynamic",
-            type=str,
-            choices=["dynamic", "lightweight"],
-            help="type of convolution",
-        )
-        parser.add_argument(
-            "--decoder-conv-type",
-            default="dynamic",
-            type=str,
-            choices=["dynamic", "lightweight"],
-            help="type of convolution",
-        )
-        parser.add_argument("--weight-softmax", default=True, type=utils.eval_bool)
-        parser.add_argument(
-            "--weight-dropout",
-            type=float,
-            metavar="D",
-            help="dropout probability for conv weights",
-        )
-
-    @classmethod
-    def build_model(cls, args, task):
-        """Build a new model instance."""
-
-        # make sure all arguments are present in older models
-        base_architecture(args)
-
-        if not safe_hasattr(args, "max_source_positions"):
-            args.max_source_positions = 1024
-        if not safe_hasattr(args, "max_target_positions"):
-            args.max_target_positions = 1024
-
-        src_dict, tgt_dict = task.source_dictionary, task.target_dictionary
-
-        def build_embedding(dictionary, embed_dim, path=None):
-            num_embeddings = len(dictionary)
-            padding_idx = dictionary.pad()
-            emb = Embedding(num_embeddings, embed_dim, padding_idx)
-            # if provided, load from preloaded dictionaries
-            if path:
-                embed_dict = utils.parse_embedding(path)
-                utils.load_embedding(embed_dict, dictionary, emb)
-            return emb
-
-        if args.share_all_embeddings:
-            if src_dict != tgt_dict:
-                raise RuntimeError(
-                    "--share-all-embeddings requires a joined dictionary"
-                )
-            if args.encoder_embed_dim != args.decoder_embed_dim:
-                raise RuntimeError(
-                    "--share-all-embeddings requires --encoder-embed-dim to match --decoder-embed-dim"
-                )
-            if args.decoder_embed_path and (
-                args.decoder_embed_path != args.encoder_embed_path
-            ):
-                raise RuntimeError(
-                    "--share-all-embeddings not compatible with --decoder-embed-path"
-                )
-            encoder_embed_tokens = build_embedding(
-                src_dict, args.encoder_embed_dim, args.encoder_embed_path
-            )
-            decoder_embed_tokens = encoder_embed_tokens
-            args.share_decoder_input_output_embed = True
-        else:
-            encoder_embed_tokens = build_embedding(
-                src_dict, args.encoder_embed_dim, args.encoder_embed_path
-            )
-            decoder_embed_tokens = build_embedding(
-                tgt_dict, args.decoder_embed_dim, args.decoder_embed_path
-            )
-
-        encoder = LightConvEncoder(args, src_dict, encoder_embed_tokens)
-        decoder = LightConvDecoder(args, tgt_dict, decoder_embed_tokens)
-        return LightConvModel(encoder, decoder)
-
-
-class LightConvEncoder(FairseqEncoder):
-    """
-    LightConv encoder consisting of *args.encoder_layers* layers. Each layer
-    is a :class:`LightConvEncoderLayer`.
-
-    Args:
-        args (argparse.Namespace): parsed command-line arguments
-        dictionary (~fairseq.data.Dictionary): encoding dictionary
-        embed_tokens (torch.nn.Embedding): input embedding
-    """
-
-    def __init__(self, args, dictionary, embed_tokens):
-        super().__init__(dictionary)
-        self.dropout_module = FairseqDropout(
-            args.dropout, module_name=self.__class__.__name__
-        )
-
-        embed_dim = embed_tokens.embedding_dim
-        self.padding_idx = embed_tokens.padding_idx
-        self.max_source_positions = args.max_source_positions
-
-        self.embed_tokens = embed_tokens
-        self.embed_scale = math.sqrt(embed_dim)
-        self.embed_positions = (
-            PositionalEmbedding(
-                args.max_source_positions,
-                embed_dim,
-                self.padding_idx,
-                learned=args.encoder_learned_pos,
-            )
-            if not args.no_token_positional_embeddings
-            else None
-        )
-
-        self.layers = nn.ModuleList([])
-        self.layers.extend(
-            [
-                LightConvEncoderLayer(
-                    args, kernel_size=args.encoder_kernel_size_list[i]
-                )
-                for i in range(args.encoder_layers)
-            ]
-        )
-        self.register_buffer("version", torch.Tensor([2]))
-        self.normalize = args.encoder_normalize_before
-        if self.normalize:
-            self.layer_norm = LayerNorm(embed_dim)
-
-    def forward(self, src_tokens, **unused):
-        """
-        Args:
-            src_tokens (LongTensor): tokens in the source language of shape
-                `(batch, src_len)`
-
-        Returns:
-            dict:
-                - **encoder_out** (Tensor): the last encoder layer's output of
-                  shape `(src_len, batch, embed_dim)`
-                - **encoder_padding_mask** (ByteTensor): the positions of
-                  padding elements of shape `(batch, src_len)`
-        """
-        # embed tokens and positions
-        x = self.embed_scale * self.embed_tokens(src_tokens)
-        if self.embed_positions is not None:
-            x += self.embed_positions(src_tokens)
-        x = self.dropout_module(x)
-
-        # B x T x C -> T x B x C
-        x = x.transpose(0, 1)
-
-        # compute padding mask
-        encoder_padding_mask = src_tokens.eq(self.padding_idx)
-        if not encoder_padding_mask.any():
-            encoder_padding_mask = None
-
-        # encoder layers
-        for layer in self.layers:
-            x = layer(x, encoder_padding_mask)
-
-        if self.normalize:
-            x = self.layer_norm(x)
-
-        return {
-            "encoder_out": x,  # T x B x C
-            "encoder_padding_mask": encoder_padding_mask,  # B x T
-        }
-
-    def reorder_encoder_out(self, encoder_out, new_order):
-        """
-        Reorder encoder output according to *new_order*.
-
-        Args:
-            encoder_out: output from the ``forward()`` method
-            new_order (LongTensor): desired order
-
-        Returns:
-            *encoder_out* rearranged according to *new_order*
-        """
-        if encoder_out["encoder_out"] is not None:
-            encoder_out["encoder_out"] = encoder_out["encoder_out"].index_select(
-                1, new_order
-            )
-        if encoder_out["encoder_padding_mask"] is not None:
-            encoder_out["encoder_padding_mask"] = encoder_out[
-                "encoder_padding_mask"
-            ].index_select(0, new_order)
-        return encoder_out
-
-    def max_positions(self):
-        """Maximum input length supported by the encoder."""
-        if self.embed_positions is None:
-            return self.max_source_positions
-        return min(self.max_source_positions, self.embed_positions.max_positions)
-
-
-class LightConvDecoder(FairseqIncrementalDecoder):
-    """
-    LightConv decoder consisting of *args.decoder_layers* layers. Each layer
-    is a :class:`LightConvDecoderLayer`.
-
-    Args:
-        args (argparse.Namespace): parsed command-line arguments
-        dictionary (~fairseq.data.Dictionary): decoding dictionary
-        embed_tokens (torch.nn.Embedding): output embedding
-        no_encoder_attn (bool, optional): whether to attend to encoder outputs.
-            Default: ``False``
-    """
-
-    def __init__(
-        self, args, dictionary, embed_tokens, no_encoder_attn=False, final_norm=True
-    ):
-        super().__init__(dictionary)
-        self.dropout_module = FairseqDropout(
-            args.dropout, module_name=self.__class__.__name__
-        )
-        self.share_input_output_embed = args.share_decoder_input_output_embed
-
-        input_embed_dim = embed_tokens.embedding_dim
-        embed_dim = args.decoder_embed_dim
-        output_embed_dim = args.decoder_output_dim
-
-        padding_idx = embed_tokens.padding_idx
-        self.max_target_positions = args.max_target_positions
-
-        self.embed_tokens = embed_tokens
-        self.embed_scale = math.sqrt(embed_dim)  # todo: try with input_embed_dim
-
-        self.project_in_dim = (
-            Linear(input_embed_dim, embed_dim, bias=False)
-            if embed_dim != input_embed_dim
-            else None
-        )
-
-        self.embed_positions = (
-            PositionalEmbedding(
-                args.max_target_positions,
-                embed_dim,
-                padding_idx,
-                learned=args.decoder_learned_pos,
-            )
-            if not args.no_token_positional_embeddings
-            else None
-        )
-
-        self.layers = nn.ModuleList([])
-        self.layers.extend(
-            [
-                LightConvDecoderLayer(
-                    args, no_encoder_attn, kernel_size=args.decoder_kernel_size_list[i]
-                )
-                for i in range(args.decoder_layers)
-            ]
-        )
-
-        self.adaptive_softmax = None
-
-        self.project_out_dim = (
-            Linear(embed_dim, output_embed_dim, bias=False)
-            if embed_dim != output_embed_dim and not args.tie_adaptive_weights
-            else None
-        )
-
-        if args.adaptive_softmax_cutoff is not None:
-            self.adaptive_softmax = AdaptiveSoftmax(
-                len(dictionary),
-                output_embed_dim,
-                utils.eval_str_list(args.adaptive_softmax_cutoff, type=int),
-                dropout=args.adaptive_softmax_dropout,
-                adaptive_inputs=embed_tokens if args.tie_adaptive_weights else None,
-                factor=args.adaptive_softmax_factor,
-                tie_proj=args.tie_adaptive_proj,
-            )
-        elif not self.share_input_output_embed:
-            self.embed_out = nn.Parameter(
-                torch.Tensor(len(dictionary), output_embed_dim)
-            )
-            nn.init.normal_(self.embed_out, mean=0, std=output_embed_dim**-0.5)
-        self.register_buffer("version", torch.Tensor([2]))
-        self.normalize = args.decoder_normalize_before and final_norm
-        if self.normalize:
-            self.layer_norm = LayerNorm(embed_dim)
-
-    def forward(
-        self, prev_output_tokens, encoder_out=None, incremental_state=None, **kwargs
-    ):
-        """
-        Args:
-            prev_output_tokens (LongTensor): previous decoder outputs of shape
-                `(batch, tgt_len)`, for teacher forcing
-            encoder_out (Tensor, optional): output from the encoder, used for
-                encoder-side attention
-            incremental_state (dict): dictionary used for storing state during
-                :ref:`Incremental decoding`
-
-        Returns:
-            tuple:
-                - the last decoder layer's output of shape `(batch, tgt_len,
-                  vocab)`
-                - the last decoder layer's attention weights of shape `(batch,
-                  tgt_len, src_len)`
-        """
-        # embed positions
-        positions = (
-            self.embed_positions(
-                prev_output_tokens,
-                incremental_state=incremental_state,
-            )
-            if self.embed_positions is not None
-            else None
-        )
-
-        if incremental_state is not None:
-            prev_output_tokens = prev_output_tokens[:, -1:]
-            if positions is not None:
-                positions = positions[:, -1:]
-
-        # embed tokens and positions
-        x = self.embed_scale * self.embed_tokens(prev_output_tokens)
-
-        if self.project_in_dim is not None:
-            x = self.project_in_dim(x)
-
-        if positions is not None:
-            x += positions
-        x = self.dropout_module(x)
-
-        # B x T x C -> T x B x C
-        x = x.transpose(0, 1)
-        attn = None
-
-        inner_states = [x]
-
-        # decoder layers
-        for layer in self.layers:
-            x, attn = layer(
-                x,
-                encoder_out["encoder_out"] if encoder_out is not None else None,
-                encoder_out["encoder_padding_mask"]
-                if encoder_out is not None
-                else None,
-                incremental_state,
-            )
-            inner_states.append(x)
-
-        if self.normalize:
-            x = self.layer_norm(x)
-
-        # T x B x C -> B x T x C
-        x = x.transpose(0, 1)
-
-        if self.project_out_dim is not None:
-            x = self.project_out_dim(x)
-
-        if self.adaptive_softmax is None:
-            # project back to size of vocabulary
-            if self.share_input_output_embed:
-                x = F.linear(x, self.embed_tokens.weight)
-            else:
-                x = F.linear(x, self.embed_out)
-
-        return x, {"attn": attn, "inner_states": inner_states}
-
-    def max_positions(self):
-        """Maximum output length supported by the decoder."""
-        if self.embed_positions is None:
-            return self.max_target_positions
-        return min(self.max_target_positions, self.embed_positions.max_positions)
-
-    def buffered_future_mask(self, tensor):
-        dim = tensor.size(0)
-        if (
-            not hasattr(self, "_future_mask")
-            or self._future_mask is None
-            or self._future_mask.device != tensor.device
-        ):
-            self._future_mask = torch.triu(
-                utils.fill_with_neg_inf(tensor.new(dim, dim)), 1
-            )
-        if self._future_mask.size(0) < dim:
-            self._future_mask = torch.triu(
-                utils.fill_with_neg_inf(self._future_mask.resize_(dim, dim)), 1
-            )
-        return self._future_mask[:dim, :dim]
-
-
-class LightConvEncoderLayer(nn.Module):
-    """Encoder layer block.
-
-    Args:
-        args (argparse.Namespace): parsed command-line arguments
-        kernel_size: kernel size of the convolution
-    """
-
-    def __init__(self, args, kernel_size=0):
-        super().__init__()
-        self.embed_dim = args.encoder_embed_dim
-        self.conv_dim = args.encoder_conv_dim
-        padding_l = (
-            kernel_size // 2
-            if kernel_size % 2 == 1
-            else ((kernel_size - 1) // 2, kernel_size // 2)
-        )
-
-        if args.encoder_glu:
-            self.linear1 = Linear(self.embed_dim, 2 * self.conv_dim)
-            self.act = nn.GLU()
-        else:
-            self.linear1 = Linear(self.embed_dim, self.conv_dim)
-            self.act = None
-        if args.encoder_conv_type == "lightweight":
-            self.conv = LightweightConv(
-                self.conv_dim,
-                kernel_size,
-                padding_l=padding_l,
-                weight_softmax=args.weight_softmax,
-                num_heads=args.encoder_attention_heads,
-                weight_dropout=args.weight_dropout,
-            )
-        elif args.encoder_conv_type == "dynamic":
-            self.conv = DynamicConv(
-                self.conv_dim,
-                kernel_size,
-                padding_l=padding_l,
-                weight_softmax=args.weight_softmax,
-                num_heads=args.encoder_attention_heads,
-                weight_dropout=args.weight_dropout,
-            )
-        else:
-            raise NotImplementedError
-        self.linear2 = Linear(self.conv_dim, self.embed_dim)
-
-        self.dropout_module = FairseqDropout(
-            args.dropout, module_name=self.__class__.__name__
-        )
-        self.relu_dropout_module = FairseqDropout(
-            args.relu_dropout, module_name=self.__class__.__name__
-        )
-        self.input_dropout_module = FairseqDropout(
-            args.input_dropout, module_name=self.__class__.__name__
-        )
-        self.normalize_before = args.encoder_normalize_before
-        self.fc1 = Linear(self.embed_dim, args.encoder_ffn_embed_dim)
-        self.fc2 = Linear(args.encoder_ffn_embed_dim, self.embed_dim)
-        self.layer_norms = nn.ModuleList([LayerNorm(self.embed_dim) for _ in range(2)])
-
-    def forward(self, x, encoder_padding_mask):
-        """
-        Args:
-            x (Tensor): input to the layer of shape `(seq_len, batch, embed_dim)`
-            encoder_padding_mask (ByteTensor): binary ByteTensor of shape
-                `(batch, src_len)` where padding elements are indicated by ``1``.
-
-        Returns:
-            encoded output of shape `(batch, src_len, embed_dim)`
-        """
-        residual = x
-        x = self.maybe_layer_norm(0, x, before=True)
-        x = self.input_dropout_module(x)
-        x = self.linear1(x)
-        if self.act is not None:
-            x = self.act(x)
-        if encoder_padding_mask is not None:
-            x = x.masked_fill(encoder_padding_mask.transpose(0, 1).unsqueeze(2), 0)
-        x = self.conv(x)
-        x = self.linear2(x)
-        x = self.dropout_module(x)
-        x = residual + x
-        x = self.maybe_layer_norm(0, x, after=True)
-
-        residual = x
-        x = self.maybe_layer_norm(1, x, before=True)
-        x = F.relu(self.fc1(x))
-        x = self.relu_dropout_module(x)
-        x = self.fc2(x)
-        x = self.dropout_module(x)
-        x = residual + x
-        x = self.maybe_layer_norm(1, x, after=True)
-        return x
-
-    def maybe_layer_norm(self, i, x, before=False, after=False):
-        assert before ^ after
-        if after ^ self.normalize_before:
-            return self.layer_norms[i](x)
-        else:
-            return x
-
-    def extra_repr(self):
-        return (
-            "dropout={}, relu_dropout={}, input_dropout={}, normalize_before={}".format(
-                self.dropout_module.p,
-                self.relu_dropout_module.p,
-                self.input_dropout_module.p,
-                self.normalize_before,
-            )
-        )
-
-
-class LightConvDecoderLayer(nn.Module):
-    """Decoder layer block.
-
-    Args:
-        args (argparse.Namespace): parsed command-line arguments
-        no_encoder_attn (bool, optional): whether to attend to encoder outputs.
-            Default: ``False``
-        kernel_size: kernel size of the convolution
-    """
-
-    def __init__(self, args, no_encoder_attn=False, kernel_size=0):
-        super().__init__()
-        self.embed_dim = args.decoder_embed_dim
-        self.conv_dim = args.decoder_conv_dim
-        if args.decoder_glu:
-            self.linear1 = Linear(self.embed_dim, 2 * self.conv_dim)
-            self.act = nn.GLU()
-        else:
-            self.linear1 = Linear(self.embed_dim, self.conv_dim)
-            self.act = None
-        if args.decoder_conv_type == "lightweight":
-            self.conv = LightweightConv(
-                self.conv_dim,
-                kernel_size,
-                padding_l=kernel_size - 1,
-                weight_softmax=args.weight_softmax,
-                num_heads=args.decoder_attention_heads,
-                weight_dropout=args.weight_dropout,
-            )
-        elif args.decoder_conv_type == "dynamic":
-            self.conv = DynamicConv(
-                self.conv_dim,
-                kernel_size,
-                padding_l=kernel_size - 1,
-                weight_softmax=args.weight_softmax,
-                num_heads=args.decoder_attention_heads,
-                weight_dropout=args.weight_dropout,
-            )
-        else:
-            raise NotImplementedError
-        self.linear2 = Linear(self.conv_dim, self.embed_dim)
-
-        self.dropout_module = FairseqDropout(
-            args.dropout, module_name=self.__class__.__name__
-        )
-        self.relu_dropout_module = FairseqDropout(
-            args.relu_dropout, module_name=self.__class__.__name__
-        )
-        self.input_dropout_module = FairseqDropout(
-            args.input_dropout, module_name=self.__class__.__name__
-        )
-        self.normalize_before = args.decoder_normalize_before
-
-        self.conv_layer_norm = LayerNorm(self.embed_dim)
-
-        if no_encoder_attn:
-            self.encoder_attn = None
-            self.encoder_attn_layer_norm = None
-        else:
-            self.encoder_attn = MultiheadAttention(
-                self.embed_dim,
-                args.decoder_attention_heads,
-                dropout=args.attention_dropout,
-                encoder_decoder_attention=True,
-            )
-            self.encoder_attn_layer_norm = LayerNorm(self.embed_dim)
-
-        self.fc1 = Linear(self.embed_dim, args.decoder_ffn_embed_dim)
-        self.fc2 = Linear(args.decoder_ffn_embed_dim, self.embed_dim)
-
-        self.final_layer_norm = LayerNorm(self.embed_dim)
-        self.need_attn = True
-
-    def forward(
-        self,
-        x,
-        encoder_out,
-        encoder_padding_mask,
-        incremental_state,
-        prev_conv_state=None,
-        prev_attn_state=None,
-        conv_mask=None,
-        conv_padding_mask=None,
-    ):
-        """
-        Args:
-            x (Tensor): input to the layer of shape `(seq_len, batch, embed_dim)`
-            encoder_padding_mask (ByteTensor): binary ByteTensor of shape
-                `(batch, src_len)` where padding elements are indicated by ``1``.
-
-        Returns:
-            encoded output of shape `(batch, src_len, embed_dim)`
-        """
-        residual = x
-        x = self.maybe_layer_norm(self.conv_layer_norm, x, before=True)
-        if prev_conv_state is not None:
-            if incremental_state is None:
-                incremental_state = {}
-            self.conv._set_input_buffer(incremental_state, prev_conv_state)
-        x = self.input_dropout_module(x)
-        x = self.linear1(x)
-        if self.act is not None:
-            x = self.act(x)
-        x = self.conv(x, incremental_state=incremental_state)
-        x = self.linear2(x)
-        x = self.dropout_module(x)
-        x = residual + x
-        x = self.maybe_layer_norm(self.conv_layer_norm, x, after=True)
-
-        attn = None
-        if self.encoder_attn is not None:
-            residual = x
-            x = self.maybe_layer_norm(self.encoder_attn_layer_norm, x, before=True)
-            if prev_attn_state is not None:
-                if incremental_state is None:
-                    incremental_state = {}
-                prev_key, prev_value = prev_attn_state
-                saved_state = {"prev_key": prev_key, "prev_value": prev_value}
-                self.encoder_attn._set_input_buffer(incremental_state, saved_state)
-            x, attn = self.encoder_attn(
-                query=x,
-                key=encoder_out,
-                value=encoder_out,
-                key_padding_mask=encoder_padding_mask,
-                incremental_state=incremental_state,
-                static_kv=True,
-                need_weights=(not self.training and self.need_attn),
-            )
-            x = self.dropout_module(x)
-            x = residual + x
-            x = self.maybe_layer_norm(self.encoder_attn_layer_norm, x, after=True)
-
-        residual = x
-        x = self.maybe_layer_norm(self.final_layer_norm, x, before=True)
-        x = F.relu(self.fc1(x))
-        x = self.relu_dropout_module(x)
-        x = self.fc2(x)
-        x = self.dropout_module(x)
-        x = residual + x
-        x = self.maybe_layer_norm(self.final_layer_norm, x, after=True)
-        return x, attn
-
-    def maybe_layer_norm(self, layer_norm, x, before=False, after=False):
-        assert before ^ after
-        if after ^ self.normalize_before:
-            return layer_norm(x)
-        else:
-            return x
-
-    def make_generation_fast_(self, need_attn=False, **kwargs):
-        self.need_attn = need_attn
-
-    def extra_repr(self):
-        return (
-            "dropout={}, relu_dropout={}, input_dropout={}, normalize_before={}".format(
-                self.dropout_module.p,
-                self.relu_dropout_module.p,
-                self.input_dropout_module.p,
-                self.normalize_before,
-            )
-        )
-
-
-def Embedding(num_embeddings, embedding_dim, padding_idx):
-    m = nn.Embedding(num_embeddings, embedding_dim, padding_idx=padding_idx)
-    nn.init.normal_(m.weight, mean=0, std=embedding_dim**-0.5)
-    nn.init.constant_(m.weight[padding_idx], 0)
-    return m
-
-
-def Linear(in_features, out_features, bias=True):
-    m = nn.Linear(in_features, out_features, bias)
-    nn.init.xavier_uniform_(m.weight)
-    if bias:
-        nn.init.constant_(m.bias, 0.0)
-    return m
-
-
-@register_model_architecture("lightconv", "lightconv")
-def base_architecture(args):
-    args.encoder_embed_path = getattr(args, "encoder_embed_path", None)
-    args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 512)
-    args.encoder_ffn_embed_dim = getattr(args, "encoder_ffn_embed_dim", 2048)
-    args.encoder_layers = getattr(args, "encoder_layers", 7)
-    args.encoder_attention_heads = getattr(args, "encoder_attention_heads", 8)
-    args.encoder_normalize_before = getattr(args, "encoder_normalize_before", False)
-    args.encoder_learned_pos = getattr(args, "encoder_learned_pos", False)
-    args.decoder_embed_path = getattr(args, "decoder_embed_path", None)
-    args.decoder_embed_dim = getattr(args, "decoder_embed_dim", args.encoder_embed_dim)
-    args.decoder_ffn_embed_dim = getattr(
-        args, "decoder_ffn_embed_dim", args.encoder_ffn_embed_dim
-    )
-    args.decoder_layers = getattr(args, "decoder_layers", 6)
-    args.decoder_attention_heads = getattr(args, "decoder_attention_heads", 8)
-    args.decoder_normalize_before = getattr(args, "decoder_normalize_before", False)
-    args.decoder_learned_pos = getattr(args, "decoder_learned_pos", False)
-    args.attention_dropout = getattr(args, "attention_dropout", 0.0)
-    args.relu_dropout = getattr(args, "relu_dropout", 0.0)
-    args.dropout = getattr(args, "dropout", 0.1)
-    args.adaptive_softmax_cutoff = getattr(args, "adaptive_softmax_cutoff", None)
-    args.adaptive_softmax_dropout = getattr(args, "adaptive_softmax_dropout", 0)
-    args.share_decoder_input_output_embed = getattr(
-        args, "share_decoder_input_output_embed", False
-    )
-    args.share_all_embeddings = getattr(args, "share_all_embeddings", False)
-    args.no_token_positional_embeddings = getattr(
-        args, "no_token_positional_embeddings", False
-    )
-
-    args.decoder_output_dim = getattr(
-        args, "decoder_output_dim", args.decoder_embed_dim
-    )
-    args.decoder_input_dim = getattr(args, "decoder_input_dim", args.decoder_embed_dim)
-
-    args.encoder_conv_dim = getattr(args, "encoder_conv_dim", args.encoder_embed_dim)
-    args.decoder_conv_dim = getattr(args, "decoder_conv_dim", args.decoder_embed_dim)
-
-    args.encoder_kernel_size_list = getattr(
-        args, "encoder_kernel_size_list", [3, 7, 15, 31, 31, 31, 31]
-    )
-    args.decoder_kernel_size_list = getattr(
-        args, "decoder_kernel_size_list", [3, 7, 15, 31, 31, 31]
-    )
-    if len(args.encoder_kernel_size_list) == 1:
-        args.encoder_kernel_size_list = (
-            args.encoder_kernel_size_list * args.encoder_layers
-        )
-    if len(args.decoder_kernel_size_list) == 1:
-        args.decoder_kernel_size_list = (
-            args.decoder_kernel_size_list * args.decoder_layers
-        )
-    assert (
-        len(args.encoder_kernel_size_list) == args.encoder_layers
-    ), "encoder_kernel_size_list doesn't match encoder_layers"
-    assert (
-        len(args.decoder_kernel_size_list) == args.decoder_layers
-    ), "decoder_kernel_size_list doesn't match decoder_layers"
-    args.encoder_glu = getattr(args, "encoder_glu", True)
-    args.decoder_glu = getattr(args, "decoder_glu", True)
-    args.input_dropout = getattr(args, "input_dropout", 0.1)
-    args.weight_dropout = getattr(args, "weight_dropout", args.attention_dropout)
-
-
-@register_model_architecture("lightconv", "lightconv_iwslt_de_en")
-def lightconv_iwslt_de_en(args):
-    args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 512)
-    args.encoder_ffn_embed_dim = getattr(args, "encoder_ffn_embed_dim", 1024)
-    args.encoder_attention_heads = getattr(args, "encoder_attention_heads", 4)
-    args.encoder_layers = getattr(args, "encoder_layers", 7)
-    args.decoder_embed_dim = getattr(args, "decoder_embed_dim", 512)
-    args.decoder_ffn_embed_dim = getattr(args, "decoder_ffn_embed_dim", 1024)
-    args.decoder_attention_heads = getattr(args, "decoder_attention_heads", 4)
-    args.decoder_layers = getattr(args, "decoder_layers", 6)
-    args.attention_dropout = getattr(args, "attention_dropout", 0.1)
-    args.weight_dropout = getattr(args, "weight_dropout", 0.1)
-    args.encoder_glu = getattr(args, "encoder_glu", False)
-    args.decoder_glu = getattr(args, "decoder_glu", False)
-    args.input_dropout = getattr(args, "input_dropout", 0.0)
-    base_architecture(args)
-
-
-@register_model_architecture("lightconv", "lightconv_wmt_en_de")
-def lightconv_wmt_en_de(args):
-    base_architecture(args)
-
-
-@register_model_architecture("lightconv", "lightconv_wmt_en_de_big")
-def lightconv_wmt_en_de_big(args):
-    args.attention_dropout = getattr(args, "attention_dropout", 0.1)
-    args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 1024)
-    args.encoder_ffn_embed_dim = getattr(args, "encoder_ffn_embed_dim", 4096)
-    args.encoder_attention_heads = getattr(args, "encoder_attention_heads", 16)
-    args.encoder_normalize_before = getattr(args, "encoder_normalize_before", False)
-    args.decoder_embed_dim = getattr(args, "decoder_embed_dim", 1024)
-    args.decoder_ffn_embed_dim = getattr(args, "decoder_ffn_embed_dim", 4096)
-    args.decoder_attention_heads = getattr(args, "decoder_attention_heads", 16)
-    args.dropout = getattr(args, "dropout", 0.3)
-    base_architecture(args)
-
-
-@register_model_architecture("lightconv", "lightconv_wmt_en_fr_big")
-def lightconv_wmt_en_fr_big(args):
-    args.dropout = getattr(args, "dropout", 0.1)
-    lightconv_wmt_en_de_big(args)
-
-
-@register_model_architecture("lightconv", "lightconv_wmt_zh_en_big")
-def lightconv_wmt_zh_en_big(args):
-    args.dropout = getattr(args, "dropout", 0.2)
-    args.attention_dropout = getattr(args, "attention_dropout", 0.2)
-    args.weight_dropout = getattr(args, "weight_dropout", 0.2)
-    lightconv_wmt_en_de_big(args)

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/lightconv_lm.py

@@ -1,306 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-from fairseq import utils
-from fairseq.models import (
-    FairseqLanguageModel,
-    register_model,
-    register_model_architecture,
-)
-from fairseq.models.lightconv import Embedding, LightConvDecoder
-from fairseq.modules import AdaptiveInput, CharacterTokenEmbedder
-
-
-@register_model("lightconv_lm")
-class LightConvLanguageModel(FairseqLanguageModel):
-    def __init__(self, decoder):
-        super().__init__(decoder)
-
-    @staticmethod
-    def add_args(parser):
-        """Add model-specific arguments to the parser."""
-        parser.add_argument(
-            "--dropout",
-            default=0.1,
-            type=float,
-            metavar="D",
-            help="dropout probability",
-        )
-        parser.add_argument(
-            "--attention-dropout",
-            default=0.0,
-            type=float,
-            metavar="D",
-            help="dropout probability for attention weights",
-        )
-        parser.add_argument(
-            "--relu-dropout",
-            default=0.0,
-            type=float,
-            metavar="D",
-            help="dropout probability after ReLU in FFN",
-        )
-        parser.add_argument(
-            "--input-dropout",
-            type=float,
-            metavar="D",
-            help="dropout probability of the inputs",
-        )
-        parser.add_argument(
-            "--decoder-embed-dim",
-            type=int,
-            metavar="N",
-            help="decoder embedding dimension",
-        )
-        parser.add_argument(
-            "--decoder-output-dim",
-            type=int,
-            metavar="N",
-            help="decoder output dimension",
-        )
-        parser.add_argument(
-            "--decoder-input-dim", type=int, metavar="N", help="decoder input dimension"
-        )
-        parser.add_argument(
-            "--decoder-ffn-embed-dim",
-            type=int,
-            metavar="N",
-            help="decoder embedding dimension for FFN",
-        )
-        parser.add_argument(
-            "--decoder-layers", type=int, metavar="N", help="num decoder layers"
-        )
-        parser.add_argument(
-            "--decoder-attention-heads",
-            type=int,
-            metavar="N",
-            help="num decoder attention heads or LightConv/DynamicConv heads",
-        )
-        parser.add_argument(
-            "--decoder-normalize-before",
-            default=False,
-            action="store_true",
-            help="apply layernorm before each decoder block",
-        )
-        parser.add_argument(
-            "--adaptive-softmax-cutoff",
-            metavar="EXPR",
-            help="comma separated list of adaptive softmax cutoff points. "
-            "Must be used with adaptive_loss criterion",
-        )
-        parser.add_argument(
-            "--adaptive-softmax-dropout",
-            type=float,
-            metavar="D",
-            help="sets adaptive softmax dropout for the tail projections",
-        )
-        parser.add_argument(
-            "--adaptive-softmax-factor",
-            type=float,
-            metavar="N",
-            help="adaptive input factor",
-        )
-        parser.add_argument(
-            "--no-token-positional-embeddings",
-            default=False,
-            action="store_true",
-            help="if set, disables positional embeddings (outside self attention)",
-        )
-        parser.add_argument(
-            "--share-decoder-input-output-embed",
-            default=False,
-            action="store_true",
-            help="share decoder input and output embeddings",
-        )
-        parser.add_argument(
-            "--character-embeddings",
-            default=False,
-            action="store_true",
-            help="if set, uses character embedding convolutions to produce token embeddings",
-        )
-        parser.add_argument(
-            "--character-filters",
-            type=str,
-            metavar="LIST",
-            default="[(1, 64), (2, 128), (3, 192), (4, 256), (5, 256), (6, 256), (7, 256)]",
-            help="size of character embeddings",
-        )
-        parser.add_argument(
-            "--character-embedding-dim",
-            type=int,
-            metavar="N",
-            default=4,
-            help="size of character embeddings",
-        )
-        parser.add_argument(
-            "--char-embedder-highway-layers",
-            type=int,
-            metavar="N",
-            default=2,
-            help="number of highway layers for character token embeddder",
-        )
-        parser.add_argument(
-            "--adaptive-input",
-            default=False,
-            action="store_true",
-            help="if set, uses adaptive input",
-        )
-        parser.add_argument(
-            "--adaptive-input-factor",
-            type=float,
-            metavar="N",
-            help="adaptive input factor",
-        )
-        parser.add_argument(
-            "--adaptive-input-cutoff",
-            metavar="EXPR",
-            help="comma separated list of adaptive input cutoff points.",
-        )
-        parser.add_argument(
-            "--tie-adaptive-weights",
-            action="store_true",
-            help="if set, ties the weights of adaptive softmax and adaptive input",
-        )
-        parser.add_argument(
-            "--tie-adaptive-proj",
-            action="store_true",
-            help="if set, ties the projection weights of adaptive softmax and adaptive input",
-        )
-        parser.add_argument(
-            "--decoder-learned-pos",
-            action="store_true",
-            help="use learned positional embeddings in the decoder",
-        )
-
-        """LightConv and DynamicConv arguments"""
-        parser.add_argument(
-            "--decoder-kernel-size-list",
-            type=lambda x: utils.eval_str_list(x, int),
-            help='list of kernel size (default: "[3,7,15,31,31,31]")',
-        )
-        parser.add_argument(
-            "--decoder-glu", type=utils.eval_bool, help="glu after in proj"
-        )
-        parser.add_argument(
-            "--decoder-conv-type",
-            default="dynamic",
-            type=str,
-            choices=["dynamic", "lightweight"],
-            help="type of convolution",
-        )
-        parser.add_argument("--weight-softmax", default=True, type=utils.eval_bool)
-        parser.add_argument(
-            "--weight-dropout",
-            type=float,
-            metavar="D",
-            help="dropout probability for conv weights",
-        )
-
-    @classmethod
-    def build_model(cls, args, task):
-        """Build a new model instance."""
-
-        # make sure all arguments are present in older models
-        base_lm_architecture(args)
-
-        if getattr(args, "max_source_positions", None) is None:
-            args.max_source_positions = args.tokens_per_sample
-        if getattr(args, "max_target_positions", None) is None:
-            args.max_target_positions = args.tokens_per_sample
-
-        if args.character_embeddings:
-            embed_tokens = CharacterTokenEmbedder(
-                task.dictionary,
-                eval(args.character_filters),
-                args.character_embedding_dim,
-                args.decoder_embed_dim,
-                args.char_embedder_highway_layers,
-            )
-        elif args.adaptive_input:
-            embed_tokens = AdaptiveInput(
-                len(task.dictionary),
-                task.dictionary.pad(),
-                args.decoder_input_dim,
-                args.adaptive_input_factor,
-                args.decoder_embed_dim,
-                utils.eval_str_list(args.adaptive_input_cutoff, type=int),
-            )
-        else:
-            embed_tokens = Embedding(
-                len(task.dictionary), args.decoder_input_dim, task.dictionary.pad()
-            )
-
-        if args.tie_adaptive_weights:
-            assert args.adaptive_input
-            assert args.adaptive_input_factor == args.adaptive_softmax_factor
-            assert (
-                args.adaptive_softmax_cutoff == args.adaptive_input_cutoff
-            ), "{} != {}".format(
-                args.adaptive_softmax_cutoff, args.adaptive_input_cutoff
-            )
-            assert args.decoder_input_dim == args.decoder_output_dim
-
-        decoder = LightConvDecoder(
-            args,
-            task.output_dictionary,
-            embed_tokens,
-            no_encoder_attn=True,
-            final_norm=False,
-        )
-        return LightConvLanguageModel(decoder)
-
-
-@register_model_architecture("lightconv_lm", "lightconv_lm")
-def base_lm_architecture(args):
-    args.decoder_embed_dim = getattr(args, "decoder_embed_dim", 512)
-    args.decoder_ffn_embed_dim = getattr(args, "decoder_ffn_embed_dim", 2048)
-    args.decoder_layers = getattr(args, "decoder_layers", 6)
-    args.decoder_attention_heads = getattr(args, "decoder_attention_heads", 8)
-    args.adaptive_softmax_cutoff = getattr(args, "adaptive_softmax_cutoff", None)
-    args.adaptive_softmax_dropout = getattr(args, "adaptive_softmax_dropout", 0)
-    args.adaptive_softmax_factor = getattr(args, "adaptive_softmax_factor", 4)
-    args.decoder_learned_pos = getattr(args, "decoder_learned_pos", False)
-
-    args.character_embeddings = getattr(args, "character_embeddings", False)
-
-    args.decoder_output_dim = getattr(
-        args, "decoder_output_dim", args.decoder_embed_dim
-    )
-    args.decoder_input_dim = getattr(args, "decoder_input_dim", args.decoder_embed_dim)
-    args.decoder_conv_dim = getattr(args, "decoder_conv_dim", args.decoder_embed_dim)
-
-    # The model training is not stable without this
-    args.decoder_normalize_before = True
-
-    args.adaptive_input = getattr(args, "adaptive_input", False)
-    args.adaptive_input_factor = getattr(args, "adaptive_input_factor", 4)
-    args.adaptive_input_cutoff = getattr(args, "adaptive_input_cutoff", None)
-
-    args.tie_adaptive_weights = getattr(args, "tie_adaptive_weights", False)
-    args.tie_adaptive_proj = getattr(args, "tie_adaptive_proj", False)
-
-    args.decoder_kernel_size_list = getattr(
-        args, "decoder_kernel_size_list", [3, 7, 15, 31, 31, 31]
-    )
-    if len(args.decoder_kernel_size_list) == 1:
-        args.decoder_kernel_size_list = (
-            args.decoder_kernel_size_list * args.decoder_layers
-        )
-    assert (
-        len(args.decoder_kernel_size_list) == args.decoder_layers
-    ), "decoder_kernel_size_list doesn't match decoder_layers"
-    args.decoder_glu = getattr(args, "decoder_glu", True)
-    args.input_dropout = getattr(args, "input_dropout", 0.1)
-    args.weight_dropout = getattr(args, "weight_dropout", args.attention_dropout)
-
-
-@register_model_architecture("lightconv_lm", "lightconv_lm_gbw")
-def lightconv_lm_gbw(args):
-    args.decoder_embed_dim = getattr(args, "decoder_embed_dim", 512)
-    args.dropout = getattr(args, "dropout", 0.1)
-    args.attention_dropout = getattr(args, "attention_dropout", 0.1)
-    args.decoder_ffn_embed_dim = getattr(args, "decoder_ffn_embed_dim", 4096)
-    args.decoder_attention_heads = getattr(args, "decoder_attention_heads", 16)
-    base_lm_architecture(args)

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/lstm.py

@@ -1,754 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-from typing import Dict, List, Optional, Tuple
-
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-from fairseq import utils
-from fairseq.models import (
-    FairseqEncoder,
-    FairseqEncoderDecoderModel,
-    FairseqIncrementalDecoder,
-    register_model,
-    register_model_architecture,
-)
-from fairseq.modules import AdaptiveSoftmax, FairseqDropout
-from torch import Tensor
-
-DEFAULT_MAX_SOURCE_POSITIONS = 1e5
-DEFAULT_MAX_TARGET_POSITIONS = 1e5
-
-
-@register_model("lstm")
-class LSTMModel(FairseqEncoderDecoderModel):
-    def __init__(self, encoder, decoder):
-        super().__init__(encoder, decoder)
-
-    @staticmethod
-    def add_args(parser):
-        """Add model-specific arguments to the parser."""
-        # fmt: off
-        parser.add_argument('--dropout', type=float, metavar='D',
-                            help='dropout probability')
-        parser.add_argument('--encoder-embed-dim', type=int, metavar='N',
-                            help='encoder embedding dimension')
-        parser.add_argument('--encoder-embed-path', type=str, metavar='STR',
-                            help='path to pre-trained encoder embedding')
-        parser.add_argument('--encoder-freeze-embed', action='store_true',
-                            help='freeze encoder embeddings')
-        parser.add_argument('--encoder-hidden-size', type=int, metavar='N',
-                            help='encoder hidden size')
-        parser.add_argument('--encoder-layers', type=int, metavar='N',
-                            help='number of encoder layers')
-        parser.add_argument('--encoder-bidirectional', action='store_true',
-                            help='make all layers of encoder bidirectional')
-        parser.add_argument('--decoder-embed-dim', type=int, metavar='N',
-                            help='decoder embedding dimension')
-        parser.add_argument('--decoder-embed-path', type=str, metavar='STR',
-                            help='path to pre-trained decoder embedding')
-        parser.add_argument('--decoder-freeze-embed', action='store_true',
-                            help='freeze decoder embeddings')
-        parser.add_argument('--decoder-hidden-size', type=int, metavar='N',
-                            help='decoder hidden size')
-        parser.add_argument('--decoder-layers', type=int, metavar='N',
-                            help='number of decoder layers')
-        parser.add_argument('--decoder-out-embed-dim', type=int, metavar='N',
-                            help='decoder output embedding dimension')
-        parser.add_argument('--decoder-attention', type=str, metavar='BOOL',
-                            help='decoder attention')
-        parser.add_argument('--adaptive-softmax-cutoff', metavar='EXPR',
-                            help='comma separated list of adaptive softmax cutoff points. '
-                                 'Must be used with adaptive_loss criterion')
-        parser.add_argument('--share-decoder-input-output-embed', default=False,
-                            action='store_true',
-                            help='share decoder input and output embeddings')
-        parser.add_argument('--share-all-embeddings', default=False, action='store_true',
-                            help='share encoder, decoder and output embeddings'
-                                 ' (requires shared dictionary and embed dim)')
-
-        # Granular dropout settings (if not specified these default to --dropout)
-        parser.add_argument('--encoder-dropout-in', type=float, metavar='D',
-                            help='dropout probability for encoder input embedding')
-        parser.add_argument('--encoder-dropout-out', type=float, metavar='D',
-                            help='dropout probability for encoder output')
-        parser.add_argument('--decoder-dropout-in', type=float, metavar='D',
-                            help='dropout probability for decoder input embedding')
-        parser.add_argument('--decoder-dropout-out', type=float, metavar='D',
-                            help='dropout probability for decoder output')
-        # fmt: on
-
-    @classmethod
-    def build_model(cls, args, task):
-        """Build a new model instance."""
-        # make sure that all args are properly defaulted (in case there are any new ones)
-        base_architecture(args)
-
-        if args.encoder_layers != args.decoder_layers:
-            raise ValueError("--encoder-layers must match --decoder-layers")
-
-        max_source_positions = getattr(
-            args, "max_source_positions", DEFAULT_MAX_SOURCE_POSITIONS
-        )
-        max_target_positions = getattr(
-            args, "max_target_positions", DEFAULT_MAX_TARGET_POSITIONS
-        )
-
-        def load_pretrained_embedding_from_file(embed_path, dictionary, embed_dim):
-            num_embeddings = len(dictionary)
-            padding_idx = dictionary.pad()
-            embed_tokens = Embedding(num_embeddings, embed_dim, padding_idx)
-            embed_dict = utils.parse_embedding(embed_path)
-            utils.print_embed_overlap(embed_dict, dictionary)
-            return utils.load_embedding(embed_dict, dictionary, embed_tokens)
-
-        if args.encoder_embed_path:
-            pretrained_encoder_embed = load_pretrained_embedding_from_file(
-                args.encoder_embed_path, task.source_dictionary, args.encoder_embed_dim
-            )
-        else:
-            num_embeddings = len(task.source_dictionary)
-            pretrained_encoder_embed = Embedding(
-                num_embeddings, args.encoder_embed_dim, task.source_dictionary.pad()
-            )
-
-        if args.share_all_embeddings:
-            # double check all parameters combinations are valid
-            if task.source_dictionary != task.target_dictionary:
-                raise ValueError("--share-all-embeddings requires a joint dictionary")
-            if args.decoder_embed_path and (
-                args.decoder_embed_path != args.encoder_embed_path
-            ):
-                raise ValueError(
-                    "--share-all-embed not compatible with --decoder-embed-path"
-                )
-            if args.encoder_embed_dim != args.decoder_embed_dim:
-                raise ValueError(
-                    "--share-all-embeddings requires --encoder-embed-dim to "
-                    "match --decoder-embed-dim"
-                )
-            pretrained_decoder_embed = pretrained_encoder_embed
-            args.share_decoder_input_output_embed = True
-        else:
-            # separate decoder input embeddings
-            pretrained_decoder_embed = None
-            if args.decoder_embed_path:
-                pretrained_decoder_embed = load_pretrained_embedding_from_file(
-                    args.decoder_embed_path,
-                    task.target_dictionary,
-                    args.decoder_embed_dim,
-                )
-        # one last double check of parameter combinations
-        if args.share_decoder_input_output_embed and (
-            args.decoder_embed_dim != args.decoder_out_embed_dim
-        ):
-            raise ValueError(
-                "--share-decoder-input-output-embeddings requires "
-                "--decoder-embed-dim to match --decoder-out-embed-dim"
-            )
-
-        if args.encoder_freeze_embed:
-            pretrained_encoder_embed.weight.requires_grad = False
-        if args.decoder_freeze_embed:
-            pretrained_decoder_embed.weight.requires_grad = False
-
-        encoder = LSTMEncoder(
-            dictionary=task.source_dictionary,
-            embed_dim=args.encoder_embed_dim,
-            hidden_size=args.encoder_hidden_size,
-            num_layers=args.encoder_layers,
-            dropout_in=args.encoder_dropout_in,
-            dropout_out=args.encoder_dropout_out,
-            bidirectional=args.encoder_bidirectional,
-            pretrained_embed=pretrained_encoder_embed,
-            max_source_positions=max_source_positions,
-        )
-        decoder = LSTMDecoder(
-            dictionary=task.target_dictionary,
-            embed_dim=args.decoder_embed_dim,
-            hidden_size=args.decoder_hidden_size,
-            out_embed_dim=args.decoder_out_embed_dim,
-            num_layers=args.decoder_layers,
-            dropout_in=args.decoder_dropout_in,
-            dropout_out=args.decoder_dropout_out,
-            attention=utils.eval_bool(args.decoder_attention),
-            encoder_output_units=encoder.output_units,
-            pretrained_embed=pretrained_decoder_embed,
-            share_input_output_embed=args.share_decoder_input_output_embed,
-            adaptive_softmax_cutoff=(
-                utils.eval_str_list(args.adaptive_softmax_cutoff, type=int)
-                if args.criterion == "adaptive_loss"
-                else None
-            ),
-            max_target_positions=max_target_positions,
-            residuals=False,
-        )
-        return cls(encoder, decoder)
-
-    def forward(
-        self,
-        src_tokens,
-        src_lengths,
-        prev_output_tokens,
-        incremental_state: Optional[Dict[str, Dict[str, Optional[Tensor]]]] = None,
-    ):
-        encoder_out = self.encoder(src_tokens, src_lengths=src_lengths)
-        decoder_out = self.decoder(
-            prev_output_tokens,
-            encoder_out=encoder_out,
-            incremental_state=incremental_state,
-        )
-        return decoder_out
-
-
-class LSTMEncoder(FairseqEncoder):
-    """LSTM encoder."""
-
-    def __init__(
-        self,
-        dictionary,
-        embed_dim=512,
-        hidden_size=512,
-        num_layers=1,
-        dropout_in=0.1,
-        dropout_out=0.1,
-        bidirectional=False,
-        left_pad=True,
-        pretrained_embed=None,
-        padding_idx=None,
-        max_source_positions=DEFAULT_MAX_SOURCE_POSITIONS,
-    ):
-        super().__init__(dictionary)
-        self.num_layers = num_layers
-        self.dropout_in_module = FairseqDropout(
-            dropout_in * 1.0, module_name=self.__class__.__name__
-        )
-        self.dropout_out_module = FairseqDropout(
-            dropout_out * 1.0, module_name=self.__class__.__name__
-        )
-        self.bidirectional = bidirectional
-        self.hidden_size = hidden_size
-        self.max_source_positions = max_source_positions
-
-        num_embeddings = len(dictionary)
-        self.padding_idx = padding_idx if padding_idx is not None else dictionary.pad()
-        if pretrained_embed is None:
-            self.embed_tokens = Embedding(num_embeddings, embed_dim, self.padding_idx)
-        else:
-            self.embed_tokens = pretrained_embed
-
-        self.lstm = LSTM(
-            input_size=embed_dim,
-            hidden_size=hidden_size,
-            num_layers=num_layers,
-            dropout=self.dropout_out_module.p if num_layers > 1 else 0.0,
-            bidirectional=bidirectional,
-        )
-        self.left_pad = left_pad
-
-        self.output_units = hidden_size
-        if bidirectional:
-            self.output_units *= 2
-
-    def forward(
-        self,
-        src_tokens: Tensor,
-        src_lengths: Tensor,
-        enforce_sorted: bool = True,
-    ):
-        """
-        Args:
-            src_tokens (LongTensor): tokens in the source language of
-                shape `(batch, src_len)`
-            src_lengths (LongTensor): lengths of each source sentence of
-                shape `(batch)`
-            enforce_sorted (bool, optional): if True, `src_tokens` is
-                expected to contain sequences sorted by length in a
-                decreasing order. If False, this condition is not
-                required. Default: True.
-        """
-        if self.left_pad:
-            # nn.utils.rnn.pack_padded_sequence requires right-padding;
-            # convert left-padding to right-padding
-            src_tokens = utils.convert_padding_direction(
-                src_tokens,
-                torch.zeros_like(src_tokens).fill_(self.padding_idx),
-                left_to_right=True,
-            )
-
-        bsz, seqlen = src_tokens.size()
-
-        # embed tokens
-        x = self.embed_tokens(src_tokens)
-        x = self.dropout_in_module(x)
-
-        # B x T x C -> T x B x C
-        x = x.transpose(0, 1)
-
-        # pack embedded source tokens into a PackedSequence
-        packed_x = nn.utils.rnn.pack_padded_sequence(
-            x, src_lengths.cpu(), enforce_sorted=enforce_sorted
-        )
-
-        # apply LSTM
-        if self.bidirectional:
-            state_size = 2 * self.num_layers, bsz, self.hidden_size
-        else:
-            state_size = self.num_layers, bsz, self.hidden_size
-        h0 = x.new_zeros(*state_size)
-        c0 = x.new_zeros(*state_size)
-        packed_outs, (final_hiddens, final_cells) = self.lstm(packed_x, (h0, c0))
-
-        # unpack outputs and apply dropout
-        x, _ = nn.utils.rnn.pad_packed_sequence(
-            packed_outs, padding_value=self.padding_idx * 1.0
-        )
-        x = self.dropout_out_module(x)
-        assert list(x.size()) == [seqlen, bsz, self.output_units]
-
-        if self.bidirectional:
-            final_hiddens = self.combine_bidir(final_hiddens, bsz)
-            final_cells = self.combine_bidir(final_cells, bsz)
-
-        encoder_padding_mask = src_tokens.eq(self.padding_idx).t()
-
-        return tuple(
-            (
-                x,  # seq_len x batch x hidden
-                final_hiddens,  # num_layers x batch x num_directions*hidden
-                final_cells,  # num_layers x batch x num_directions*hidden
-                encoder_padding_mask,  # seq_len x batch
-            )
-        )
-
-    def combine_bidir(self, outs, bsz: int):
-        out = outs.view(self.num_layers, 2, bsz, -1).transpose(1, 2).contiguous()
-        return out.view(self.num_layers, bsz, -1)
-
-    def reorder_encoder_out(
-        self, encoder_out: Tuple[Tensor, Tensor, Tensor, Tensor], new_order
-    ):
-        return tuple(
-            (
-                encoder_out[0].index_select(1, new_order),
-                encoder_out[1].index_select(1, new_order),
-                encoder_out[2].index_select(1, new_order),
-                encoder_out[3].index_select(1, new_order),
-            )
-        )
-
-    def max_positions(self):
-        """Maximum input length supported by the encoder."""
-        return self.max_source_positions
-
-
-class AttentionLayer(nn.Module):
-    def __init__(self, input_embed_dim, source_embed_dim, output_embed_dim, bias=False):
-        super().__init__()
-
-        self.input_proj = Linear(input_embed_dim, source_embed_dim, bias=bias)
-        self.output_proj = Linear(
-            input_embed_dim + source_embed_dim, output_embed_dim, bias=bias
-        )
-
-    def forward(self, input, source_hids, encoder_padding_mask):
-        # input: bsz x input_embed_dim
-        # source_hids: srclen x bsz x source_embed_dim
-
-        # x: bsz x source_embed_dim
-        x = self.input_proj(input)
-
-        # compute attention
-        attn_scores = (source_hids * x.unsqueeze(0)).sum(dim=2)
-
-        # don't attend over padding
-        if encoder_padding_mask is not None:
-            attn_scores = (
-                attn_scores.float()
-                .masked_fill_(encoder_padding_mask, float("-inf"))
-                .type_as(attn_scores)
-            )  # FP16 support: cast to float and back
-
-        attn_scores = F.softmax(attn_scores, dim=0)  # srclen x bsz
-
-        # sum weighted sources
-        x = (attn_scores.unsqueeze(2) * source_hids).sum(dim=0)
-
-        x = torch.tanh(self.output_proj(torch.cat((x, input), dim=1)))
-        return x, attn_scores
-
-
-class LSTMDecoder(FairseqIncrementalDecoder):
-    """LSTM decoder."""
-
-    def __init__(
-        self,
-        dictionary,
-        embed_dim=512,
-        hidden_size=512,
-        out_embed_dim=512,
-        num_layers=1,
-        dropout_in=0.1,
-        dropout_out=0.1,
-        attention=True,
-        encoder_output_units=512,
-        pretrained_embed=None,
-        share_input_output_embed=False,
-        adaptive_softmax_cutoff=None,
-        max_target_positions=DEFAULT_MAX_TARGET_POSITIONS,
-        residuals=False,
-    ):
-        super().__init__(dictionary)
-        self.dropout_in_module = FairseqDropout(
-            dropout_in * 1.0, module_name=self.__class__.__name__
-        )
-        self.dropout_out_module = FairseqDropout(
-            dropout_out * 1.0, module_name=self.__class__.__name__
-        )
-        self.hidden_size = hidden_size
-        self.share_input_output_embed = share_input_output_embed
-        self.need_attn = True
-        self.max_target_positions = max_target_positions
-        self.residuals = residuals
-        self.num_layers = num_layers
-
-        self.adaptive_softmax = None
-        num_embeddings = len(dictionary)
-        padding_idx = dictionary.pad()
-        if pretrained_embed is None:
-            self.embed_tokens = Embedding(num_embeddings, embed_dim, padding_idx)
-        else:
-            self.embed_tokens = pretrained_embed
-
-        self.encoder_output_units = encoder_output_units
-        if encoder_output_units != hidden_size and encoder_output_units != 0:
-            self.encoder_hidden_proj = Linear(encoder_output_units, hidden_size)
-            self.encoder_cell_proj = Linear(encoder_output_units, hidden_size)
-        else:
-            self.encoder_hidden_proj = self.encoder_cell_proj = None
-
-        # disable input feeding if there is no encoder
-        # input feeding is described in arxiv.org/abs/1508.04025
-        input_feed_size = 0 if encoder_output_units == 0 else hidden_size
-        self.layers = nn.ModuleList(
-            [
-                LSTMCell(
-                    input_size=input_feed_size + embed_dim
-                    if layer == 0
-                    else hidden_size,
-                    hidden_size=hidden_size,
-                )
-                for layer in range(num_layers)
-            ]
-        )
-
-        if attention:
-            # TODO make bias configurable
-            self.attention = AttentionLayer(
-                hidden_size, encoder_output_units, hidden_size, bias=False
-            )
-        else:
-            self.attention = None
-
-        if hidden_size != out_embed_dim:
-            self.additional_fc = Linear(hidden_size, out_embed_dim)
-
-        if adaptive_softmax_cutoff is not None:
-            # setting adaptive_softmax dropout to dropout_out for now but can be redefined
-            self.adaptive_softmax = AdaptiveSoftmax(
-                num_embeddings,
-                hidden_size,
-                adaptive_softmax_cutoff,
-                dropout=dropout_out,
-            )
-        elif not self.share_input_output_embed:
-            self.fc_out = Linear(out_embed_dim, num_embeddings, dropout=dropout_out)
-
-    def forward(
-        self,
-        prev_output_tokens,
-        encoder_out: Optional[Tuple[Tensor, Tensor, Tensor, Tensor]] = None,
-        incremental_state: Optional[Dict[str, Dict[str, Optional[Tensor]]]] = None,
-        src_lengths: Optional[Tensor] = None,
-    ):
-        x, attn_scores = self.extract_features(
-            prev_output_tokens, encoder_out, incremental_state
-        )
-        return self.output_layer(x), attn_scores
-
-    def extract_features(
-        self,
-        prev_output_tokens,
-        encoder_out: Optional[Tuple[Tensor, Tensor, Tensor, Tensor]] = None,
-        incremental_state: Optional[Dict[str, Dict[str, Optional[Tensor]]]] = None,
-    ):
-        """
-        Similar to *forward* but only return features.
-        """
-        # get outputs from encoder
-        if encoder_out is not None:
-            encoder_outs = encoder_out[0]
-            encoder_hiddens = encoder_out[1]
-            encoder_cells = encoder_out[2]
-            encoder_padding_mask = encoder_out[3]
-        else:
-            encoder_outs = torch.empty(0)
-            encoder_hiddens = torch.empty(0)
-            encoder_cells = torch.empty(0)
-            encoder_padding_mask = torch.empty(0)
-        srclen = encoder_outs.size(0)
-
-        if incremental_state is not None and len(incremental_state) > 0:
-            prev_output_tokens = prev_output_tokens[:, -1:]
-
-        bsz, seqlen = prev_output_tokens.size()
-
-        # embed tokens
-        x = self.embed_tokens(prev_output_tokens)
-        x = self.dropout_in_module(x)
-
-        # B x T x C -> T x B x C
-        x = x.transpose(0, 1)
-
-        # initialize previous states (or get from cache during incremental generation)
-        if incremental_state is not None and len(incremental_state) > 0:
-            prev_hiddens, prev_cells, input_feed = self.get_cached_state(
-                incremental_state
-            )
-        elif encoder_out is not None:
-            # setup recurrent cells
-            prev_hiddens = [encoder_hiddens[i] for i in range(self.num_layers)]
-            prev_cells = [encoder_cells[i] for i in range(self.num_layers)]
-            if self.encoder_hidden_proj is not None:
-                prev_hiddens = [self.encoder_hidden_proj(y) for y in prev_hiddens]
-                prev_cells = [self.encoder_cell_proj(y) for y in prev_cells]
-            input_feed = x.new_zeros(bsz, self.hidden_size)
-        else:
-            # setup zero cells, since there is no encoder
-            zero_state = x.new_zeros(bsz, self.hidden_size)
-            prev_hiddens = [zero_state for i in range(self.num_layers)]
-            prev_cells = [zero_state for i in range(self.num_layers)]
-            input_feed = None
-
-        assert (
-            srclen > 0 or self.attention is None
-        ), "attention is not supported if there are no encoder outputs"
-        attn_scores: Optional[Tensor] = (
-            x.new_zeros(srclen, seqlen, bsz) if self.attention is not None else None
-        )
-        outs = []
-        for j in range(seqlen):
-            # input feeding: concatenate context vector from previous time step
-            if input_feed is not None:
-                input = torch.cat((x[j, :, :], input_feed), dim=1)
-            else:
-                input = x[j]
-
-            for i, rnn in enumerate(self.layers):
-                # recurrent cell
-                hidden, cell = rnn(input, (prev_hiddens[i], prev_cells[i]))
-
-                # hidden state becomes the input to the next layer
-                input = self.dropout_out_module(hidden)
-                if self.residuals:
-                    input = input + prev_hiddens[i]
-
-                # save state for next time step
-                prev_hiddens[i] = hidden
-                prev_cells[i] = cell
-
-            # apply attention using the last layer's hidden state
-            if self.attention is not None:
-                assert attn_scores is not None
-                out, attn_scores[:, j, :] = self.attention(
-                    hidden, encoder_outs, encoder_padding_mask
-                )
-            else:
-                out = hidden
-            out = self.dropout_out_module(out)
-
-            # input feeding
-            if input_feed is not None:
-                input_feed = out
-
-            # save final output
-            outs.append(out)
-
-        # Stack all the necessary tensors together and store
-        prev_hiddens_tensor = torch.stack(prev_hiddens)
-        prev_cells_tensor = torch.stack(prev_cells)
-        cache_state = torch.jit.annotate(
-            Dict[str, Optional[Tensor]],
-            {
-                "prev_hiddens": prev_hiddens_tensor,
-                "prev_cells": prev_cells_tensor,
-                "input_feed": input_feed,
-            },
-        )
-        self.set_incremental_state(incremental_state, "cached_state", cache_state)
-
-        # collect outputs across time steps
-        x = torch.cat(outs, dim=0).view(seqlen, bsz, self.hidden_size)
-
-        # T x B x C -> B x T x C
-        x = x.transpose(1, 0)
-
-        if hasattr(self, "additional_fc") and self.adaptive_softmax is None:
-            x = self.additional_fc(x)
-            x = self.dropout_out_module(x)
-        # srclen x tgtlen x bsz -> bsz x tgtlen x srclen
-        if not self.training and self.need_attn and self.attention is not None:
-            assert attn_scores is not None
-            attn_scores = attn_scores.transpose(0, 2)
-        else:
-            attn_scores = None
-        return x, attn_scores
-
-    def output_layer(self, x):
-        """Project features to the vocabulary size."""
-        if self.adaptive_softmax is None:
-            if self.share_input_output_embed:
-                x = F.linear(x, self.embed_tokens.weight)
-            else:
-                x = self.fc_out(x)
-        return x
-
-    def get_cached_state(
-        self,
-        incremental_state: Dict[str, Dict[str, Optional[Tensor]]],
-    ) -> Tuple[List[Tensor], List[Tensor], Optional[Tensor]]:
-        cached_state = self.get_incremental_state(incremental_state, "cached_state")
-        assert cached_state is not None
-        prev_hiddens_ = cached_state["prev_hiddens"]
-        assert prev_hiddens_ is not None
-        prev_cells_ = cached_state["prev_cells"]
-        assert prev_cells_ is not None
-        prev_hiddens = [prev_hiddens_[i] for i in range(self.num_layers)]
-        prev_cells = [prev_cells_[j] for j in range(self.num_layers)]
-        input_feed = cached_state[
-            "input_feed"
-        ]  # can be None for decoder-only language models
-        return prev_hiddens, prev_cells, input_feed
-
-    def reorder_incremental_state(
-        self,
-        incremental_state: Dict[str, Dict[str, Optional[Tensor]]],
-        new_order: Tensor,
-    ):
-        if incremental_state is None or len(incremental_state) == 0:
-            return
-        prev_hiddens, prev_cells, input_feed = self.get_cached_state(incremental_state)
-        prev_hiddens = [p.index_select(0, new_order) for p in prev_hiddens]
-        prev_cells = [p.index_select(0, new_order) for p in prev_cells]
-        if input_feed is not None:
-            input_feed = input_feed.index_select(0, new_order)
-        cached_state_new = torch.jit.annotate(
-            Dict[str, Optional[Tensor]],
-            {
-                "prev_hiddens": torch.stack(prev_hiddens),
-                "prev_cells": torch.stack(prev_cells),
-                "input_feed": input_feed,
-            },
-        )
-        self.set_incremental_state(incremental_state, "cached_state", cached_state_new),
-        return
-
-    def max_positions(self):
-        """Maximum output length supported by the decoder."""
-        return self.max_target_positions
-
-    def make_generation_fast_(self, need_attn=False, **kwargs):
-        self.need_attn = need_attn
-
-
-def Embedding(num_embeddings, embedding_dim, padding_idx):
-    m = nn.Embedding(num_embeddings, embedding_dim, padding_idx=padding_idx)
-    nn.init.uniform_(m.weight, -0.1, 0.1)
-    nn.init.constant_(m.weight[padding_idx], 0)
-    return m
-
-
-def LSTM(input_size, hidden_size, **kwargs):
-    m = nn.LSTM(input_size, hidden_size, **kwargs)
-    for name, param in m.named_parameters():
-        if "weight" in name or "bias" in name:
-            param.data.uniform_(-0.1, 0.1)
-    return m
-
-
-def LSTMCell(input_size, hidden_size, **kwargs):
-    m = nn.LSTMCell(input_size, hidden_size, **kwargs)
-    for name, param in m.named_parameters():
-        if "weight" in name or "bias" in name:
-            param.data.uniform_(-0.1, 0.1)
-    return m
-
-
-def Linear(in_features, out_features, bias=True, dropout=0.0):
-    """Linear layer (input: N x T x C)"""
-    m = nn.Linear(in_features, out_features, bias=bias)
-    m.weight.data.uniform_(-0.1, 0.1)
-    if bias:
-        m.bias.data.uniform_(-0.1, 0.1)
-    return m
-
-
-@register_model_architecture("lstm", "lstm")
-def base_architecture(args):
-    args.dropout = getattr(args, "dropout", 0.1)
-    args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 512)
-    args.encoder_embed_path = getattr(args, "encoder_embed_path", None)
-    args.encoder_freeze_embed = getattr(args, "encoder_freeze_embed", False)
-    args.encoder_hidden_size = getattr(
-        args, "encoder_hidden_size", args.encoder_embed_dim
-    )
-    args.encoder_layers = getattr(args, "encoder_layers", 1)
-    args.encoder_bidirectional = getattr(args, "encoder_bidirectional", False)
-    args.encoder_dropout_in = getattr(args, "encoder_dropout_in", args.dropout)
-    args.encoder_dropout_out = getattr(args, "encoder_dropout_out", args.dropout)
-    args.decoder_embed_dim = getattr(args, "decoder_embed_dim", 512)
-    args.decoder_embed_path = getattr(args, "decoder_embed_path", None)
-    args.decoder_freeze_embed = getattr(args, "decoder_freeze_embed", False)
-    args.decoder_hidden_size = getattr(
-        args, "decoder_hidden_size", args.decoder_embed_dim
-    )
-    args.decoder_layers = getattr(args, "decoder_layers", 1)
-    args.decoder_out_embed_dim = getattr(args, "decoder_out_embed_dim", 512)
-    args.decoder_attention = getattr(args, "decoder_attention", "1")
-    args.decoder_dropout_in = getattr(args, "decoder_dropout_in", args.dropout)
-    args.decoder_dropout_out = getattr(args, "decoder_dropout_out", args.dropout)
-    args.share_decoder_input_output_embed = getattr(
-        args, "share_decoder_input_output_embed", False
-    )
-    args.share_all_embeddings = getattr(args, "share_all_embeddings", False)
-    args.adaptive_softmax_cutoff = getattr(
-        args, "adaptive_softmax_cutoff", "10000,50000,200000"
-    )
-
-
-@register_model_architecture("lstm", "lstm_wiseman_iwslt_de_en")
-def lstm_wiseman_iwslt_de_en(args):
-    args.dropout = getattr(args, "dropout", 0.1)
-    args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 256)
-    args.encoder_dropout_in = getattr(args, "encoder_dropout_in", 0)
-    args.encoder_dropout_out = getattr(args, "encoder_dropout_out", 0)
-    args.decoder_embed_dim = getattr(args, "decoder_embed_dim", 256)
-    args.decoder_out_embed_dim = getattr(args, "decoder_out_embed_dim", 256)
-    args.decoder_dropout_in = getattr(args, "decoder_dropout_in", 0)
-    args.decoder_dropout_out = getattr(args, "decoder_dropout_out", args.dropout)
-    base_architecture(args)
-
-
-@register_model_architecture("lstm", "lstm_luong_wmt_en_de")
-def lstm_luong_wmt_en_de(args):
-    args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 1000)
-    args.encoder_layers = getattr(args, "encoder_layers", 4)
-    args.encoder_dropout_out = getattr(args, "encoder_dropout_out", 0)
-    args.decoder_embed_dim = getattr(args, "decoder_embed_dim", 1000)
-    args.decoder_layers = getattr(args, "decoder_layers", 4)
-    args.decoder_out_embed_dim = getattr(args, "decoder_out_embed_dim", 1000)
-    args.decoder_dropout_out = getattr(args, "decoder_dropout_out", 0)
-    base_architecture(args)

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/lstm_lm.py

@@ -1,141 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-from fairseq import utils
-from fairseq.models import (
-    FairseqLanguageModel,
-    register_model,
-    register_model_architecture,
-)
-from fairseq.models.lstm import Embedding, LSTMDecoder
-
-DEFAULT_MAX_TARGET_POSITIONS = 1e5
-
-
-@register_model("lstm_lm")
-class LSTMLanguageModel(FairseqLanguageModel):
-    def __init__(self, decoder):
-        super().__init__(decoder)
-
-    @staticmethod
-    def add_args(parser):
-        """Add model-specific arguments to the parser."""
-        # fmt: off
-        parser.add_argument('--dropout', type=float, metavar='D',
-                            help='dropout probability')
-        parser.add_argument('--decoder-embed-dim', type=int, metavar='N',
-                            help='decoder embedding dimension')
-        parser.add_argument('--decoder-embed-path', type=str, metavar='STR',
-                            help='path to pre-trained decoder embedding')
-        parser.add_argument('--decoder-hidden-size', type=int, metavar='N',
-                            help='decoder hidden size')
-        parser.add_argument('--decoder-layers', type=int, metavar='N',
-                            help='number of decoder layers')
-        parser.add_argument('--decoder-out-embed-dim', type=int, metavar='N',
-                            help='decoder output embedding dimension')
-        parser.add_argument('--decoder-attention', type=str, metavar='BOOL',
-                            help='decoder attention')
-        parser.add_argument('--adaptive-softmax-cutoff', metavar='EXPR',
-                            help='comma separated list of adaptive softmax cutoff points. '
-                                 'Must be used with adaptive_loss criterion')
-        parser.add_argument('--residuals', default=False,
-                            action='store_true',
-                            help='applying residuals between LSTM layers')
-
-        # Granular dropout settings (if not specified these default to --dropout)
-        parser.add_argument('--decoder-dropout-in', type=float, metavar='D',
-                            help='dropout probability for decoder input embedding')
-        parser.add_argument('--decoder-dropout-out', type=float, metavar='D',
-                            help='dropout probability for decoder output')
-        parser.add_argument('--share-decoder-input-output-embed', default=False,
-                            action='store_true',
-                            help='share decoder input and output embeddings')
-        # fmt: on
-
-    @classmethod
-    def build_model(cls, args, task):
-        """Build a new model instance."""
-
-        # make sure all arguments are present in older models
-        base_architecture(args)
-
-        if getattr(args, "max_target_positions", None) is not None:
-            max_target_positions = args.max_target_positions
-        else:
-            max_target_positions = getattr(
-                args, "tokens_per_sample", DEFAULT_MAX_TARGET_POSITIONS
-            )
-
-        def load_pretrained_embedding_from_file(embed_path, dictionary, embed_dim):
-            num_embeddings = len(dictionary)
-            padding_idx = dictionary.pad()
-            embed_tokens = Embedding(num_embeddings, embed_dim, padding_idx)
-            embed_dict = utils.parse_embedding(embed_path)
-            utils.print_embed_overlap(embed_dict, dictionary)
-            return utils.load_embedding(embed_dict, dictionary, embed_tokens)
-
-        pretrained_decoder_embed = None
-        if args.decoder_embed_path:
-            pretrained_decoder_embed = load_pretrained_embedding_from_file(
-                args.decoder_embed_path, task.target_dictionary, args.decoder_embed_dim
-            )
-
-        if args.share_decoder_input_output_embed:
-            # double check all parameters combinations are valid
-            if task.source_dictionary != task.target_dictionary:
-                raise ValueError(
-                    "--share-decoder-input-output-embeddings requires a joint dictionary"
-                )
-
-            if args.decoder_embed_dim != args.decoder_out_embed_dim:
-                raise ValueError(
-                    "--share-decoder-input-output-embeddings requires "
-                    "--decoder-embed-dim to match --decoder-out-embed-dim"
-                )
-
-        decoder = LSTMDecoder(
-            dictionary=task.dictionary,
-            embed_dim=args.decoder_embed_dim,
-            hidden_size=args.decoder_hidden_size,
-            out_embed_dim=args.decoder_out_embed_dim,
-            num_layers=args.decoder_layers,
-            dropout_in=args.decoder_dropout_in,
-            dropout_out=args.decoder_dropout_out,
-            attention=False,  # decoder-only language model doesn't support attention
-            encoder_output_units=0,
-            pretrained_embed=pretrained_decoder_embed,
-            share_input_output_embed=args.share_decoder_input_output_embed,
-            adaptive_softmax_cutoff=(
-                utils.eval_str_list(args.adaptive_softmax_cutoff, type=int)
-                if args.criterion == "adaptive_loss"
-                else None
-            ),
-            max_target_positions=max_target_positions,
-            residuals=args.residuals,
-        )
-
-        return cls(decoder)
-
-
-@register_model_architecture("lstm_lm", "lstm_lm")
-def base_architecture(args):
-    args.dropout = getattr(args, "dropout", 0.1)
-    args.decoder_embed_dim = getattr(args, "decoder_embed_dim", 512)
-    args.decoder_embed_path = getattr(args, "decoder_embed_path", None)
-    args.decoder_hidden_size = getattr(
-        args, "decoder_hidden_size", args.decoder_embed_dim
-    )
-    args.decoder_layers = getattr(args, "decoder_layers", 1)
-    args.decoder_out_embed_dim = getattr(args, "decoder_out_embed_dim", 512)
-    args.decoder_attention = getattr(args, "decoder_attention", "0")
-    args.decoder_dropout_in = getattr(args, "decoder_dropout_in", args.dropout)
-    args.decoder_dropout_out = getattr(args, "decoder_dropout_out", args.dropout)
-    args.share_decoder_input_output_embed = getattr(
-        args, "share_decoder_input_output_embed", False
-    )
-    args.adaptive_softmax_cutoff = getattr(
-        args, "adaptive_softmax_cutoff", "10000,50000,200000"
-    )
-    args.residuals = getattr(args, "residuals", False)

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/masked_lm.py

@@ -1,403 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-import logging
-
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-from fairseq import utils
-from fairseq.models import (
-    FairseqEncoder,
-    FairseqEncoderModel,
-    register_model,
-    register_model_architecture,
-)
-from fairseq.modules import (
-    LayerNorm,
-    SinusoidalPositionalEmbedding,
-    TransformerSentenceEncoder,
-)
-from fairseq.modules.transformer_sentence_encoder import init_bert_params
-from fairseq.utils import safe_hasattr
-
-logger = logging.getLogger(__name__)
-
-
-@register_model("masked_lm")
-class MaskedLMModel(FairseqEncoderModel):
-    """
-    Class for training a Masked Language Model. It also supports an
-    additional sentence level prediction if the sent-loss argument is set.
-    """
-
-    def __init__(self, args, encoder):
-        super().__init__(encoder)
-        self.args = args
-
-        # if specified then apply bert initialization on the model. We need
-        # to explictly call this to make sure that the output embeddings
-        # and projection layers are also correctly initialized
-        if getattr(args, "apply_bert_init", False):
-            self.apply(init_bert_params)
-
-    @staticmethod
-    def add_args(parser):
-        """Add model-specific arguments to the parser."""
-        # Arguments related to dropout
-        parser.add_argument(
-            "--dropout", type=float, metavar="D", help="dropout probability"
-        )
-        parser.add_argument(
-            "--attention-dropout",
-            type=float,
-            metavar="D",
-            help="dropout probability for" " attention weights",
-        )
-        parser.add_argument(
-            "--act-dropout",
-            type=float,
-            metavar="D",
-            help="dropout probability after" " activation in FFN",
-        )
-
-        # Arguments related to hidden states and self-attention
-        parser.add_argument(
-            "--encoder-ffn-embed-dim",
-            type=int,
-            metavar="N",
-            help="encoder embedding dimension for FFN",
-        )
-        parser.add_argument(
-            "--encoder-layers", type=int, metavar="N", help="num encoder layers"
-        )
-        parser.add_argument(
-            "--encoder-attention-heads",
-            type=int,
-            metavar="N",
-            help="num encoder attention heads",
-        )
-
-        # Arguments related to input and output embeddings
-        parser.add_argument(
-            "--encoder-embed-dim",
-            type=int,
-            metavar="N",
-            help="encoder embedding dimension",
-        )
-        parser.add_argument(
-            "--share-encoder-input-output-embed",
-            action="store_true",
-            help="share encoder input" " and output embeddings",
-        )
-        parser.add_argument(
-            "--encoder-learned-pos",
-            action="store_true",
-            help="use learned positional embeddings in the encoder",
-        )
-        parser.add_argument(
-            "--no-token-positional-embeddings",
-            action="store_true",
-            help="if set, disables positional embeddings" " (outside self attention)",
-        )
-        parser.add_argument(
-            "--num-segment", type=int, metavar="N", help="num segment in the input"
-        )
-        parser.add_argument(
-            "--max-positions", type=int, help="number of positional embeddings to learn"
-        )
-
-        # Arguments related to sentence level prediction
-        parser.add_argument(
-            "--sentence-class-num",
-            type=int,
-            metavar="N",
-            help="number of classes for sentence task",
-        )
-        parser.add_argument(
-            "--sent-loss",
-            action="store_true",
-            help="if set," " calculate sentence level predictions",
-        )
-
-        # Arguments related to parameter initialization
-        parser.add_argument(
-            "--apply-bert-init",
-            action="store_true",
-            help="use custom param initialization for BERT",
-        )
-
-        # misc params
-        parser.add_argument(
-            "--activation-fn",
-            choices=utils.get_available_activation_fns(),
-            help="activation function to use",
-        )
-        parser.add_argument(
-            "--pooler-activation-fn",
-            choices=utils.get_available_activation_fns(),
-            help="Which activation function to use for pooler layer.",
-        )
-        parser.add_argument(
-            "--encoder-normalize-before",
-            action="store_true",
-            help="apply layernorm before each encoder block",
-        )
-
-    def forward(self, src_tokens, segment_labels=None, **kwargs):
-        return self.encoder(src_tokens, segment_labels=segment_labels, **kwargs)
-
-    def max_positions(self):
-        return self.encoder.max_positions
-
-    @classmethod
-    def build_model(cls, args, task):
-        """Build a new model instance."""
-        # make sure all arguments are present in older models
-        base_architecture(args)
-
-        if not safe_hasattr(args, "max_positions"):
-            args.max_positions = args.tokens_per_sample
-
-        logger.info(args)
-
-        encoder = MaskedLMEncoder(args, task.dictionary)
-        return cls(args, encoder)
-
-
-class MaskedLMEncoder(FairseqEncoder):
-    """
-    Encoder for Masked Language Modelling.
-    """
-
-    def __init__(self, args, dictionary):
-        super().__init__(dictionary)
-
-        self.padding_idx = dictionary.pad()
-        self.vocab_size = dictionary.__len__()
-        self.max_positions = args.max_positions
-
-        self.sentence_encoder = TransformerSentenceEncoder(
-            padding_idx=self.padding_idx,
-            vocab_size=self.vocab_size,
-            num_encoder_layers=args.encoder_layers,
-            embedding_dim=args.encoder_embed_dim,
-            ffn_embedding_dim=args.encoder_ffn_embed_dim,
-            num_attention_heads=args.encoder_attention_heads,
-            dropout=args.dropout,
-            attention_dropout=args.attention_dropout,
-            activation_dropout=args.act_dropout,
-            max_seq_len=self.max_positions,
-            num_segments=args.num_segment,
-            use_position_embeddings=not args.no_token_positional_embeddings,
-            encoder_normalize_before=args.encoder_normalize_before,
-            apply_bert_init=args.apply_bert_init,
-            activation_fn=args.activation_fn,
-            learned_pos_embedding=args.encoder_learned_pos,
-        )
-
-        self.share_input_output_embed = args.share_encoder_input_output_embed
-        self.embed_out = None
-        self.sentence_projection_layer = None
-        self.sentence_out_dim = args.sentence_class_num
-        self.lm_output_learned_bias = None
-
-        # Remove head is set to true during fine-tuning
-        self.load_softmax = not getattr(args, "remove_head", False)
-
-        self.masked_lm_pooler = nn.Linear(
-            args.encoder_embed_dim, args.encoder_embed_dim
-        )
-        self.pooler_activation = utils.get_activation_fn(args.pooler_activation_fn)
-
-        self.lm_head_transform_weight = nn.Linear(
-            args.encoder_embed_dim, args.encoder_embed_dim
-        )
-        self.activation_fn = utils.get_activation_fn(args.activation_fn)
-        self.layer_norm = LayerNorm(args.encoder_embed_dim)
-
-        self.lm_output_learned_bias = None
-        if self.load_softmax:
-            self.lm_output_learned_bias = nn.Parameter(torch.zeros(self.vocab_size))
-
-            if not self.share_input_output_embed:
-                self.embed_out = nn.Linear(
-                    args.encoder_embed_dim, self.vocab_size, bias=False
-                )
-
-            if args.sent_loss:
-                self.sentence_projection_layer = nn.Linear(
-                    args.encoder_embed_dim, self.sentence_out_dim, bias=False
-                )
-
-    def forward(self, src_tokens, segment_labels=None, masked_tokens=None, **unused):
-        """
-        Forward pass for Masked LM encoder. This first computes the token
-        embedding using the token embedding matrix, position embeddings (if
-        specified) and segment embeddings (if specified).
-
-        Here we assume that the sentence representation corresponds to the
-        output of the classification_token (see bert_task or cross_lingual_lm
-        task for more details).
-        Args:
-            - src_tokens: B x T matrix representing sentences
-            - segment_labels: B x T matrix representing segment label for tokens
-        Returns:
-            - a tuple of the following:
-                - logits for predictions in format B x T x C to be used in
-                  softmax afterwards
-                - a dictionary of additional data, where 'pooled_output' contains
-                  the representation for classification_token and 'inner_states'
-                  is a list of internal model states used to compute the
-                  predictions (similar in ELMO). 'sentence_logits'
-                  is the prediction logit for NSP task and is only computed if
-                  this is specified in the input arguments.
-        """
-
-        inner_states, sentence_rep = self.sentence_encoder(
-            src_tokens,
-            segment_labels=segment_labels,
-        )
-
-        x = inner_states[-1].transpose(0, 1)
-        # project masked tokens only
-        if masked_tokens is not None:
-            x = x[masked_tokens, :]
-        x = self.layer_norm(self.activation_fn(self.lm_head_transform_weight(x)))
-
-        pooled_output = self.pooler_activation(self.masked_lm_pooler(sentence_rep))
-
-        # project back to size of vocabulary
-        if self.share_input_output_embed and hasattr(
-            self.sentence_encoder.embed_tokens, "weight"
-        ):
-            x = F.linear(x, self.sentence_encoder.embed_tokens.weight)
-        elif self.embed_out is not None:
-            x = self.embed_out(x)
-        if self.lm_output_learned_bias is not None:
-            x = x + self.lm_output_learned_bias
-        sentence_logits = None
-        if self.sentence_projection_layer:
-            sentence_logits = self.sentence_projection_layer(pooled_output)
-
-        return x, {
-            "inner_states": inner_states,
-            "pooled_output": pooled_output,
-            "sentence_logits": sentence_logits,
-        }
-
-    def max_positions(self):
-        """Maximum output length supported by the encoder."""
-        return self.max_positions
-
-    def upgrade_state_dict_named(self, state_dict, name):
-        if isinstance(
-            self.sentence_encoder.embed_positions, SinusoidalPositionalEmbedding
-        ):
-            state_dict[
-                name + ".sentence_encoder.embed_positions._float_tensor"
-            ] = torch.FloatTensor(1)
-        if not self.load_softmax:
-            for k in list(state_dict.keys()):
-                if (
-                    "embed_out.weight" in k
-                    or "sentence_projection_layer.weight" in k
-                    or "lm_output_learned_bias" in k
-                ):
-                    del state_dict[k]
-        return state_dict
-
-
-@register_model_architecture("masked_lm", "masked_lm")
-def base_architecture(args):
-    args.dropout = getattr(args, "dropout", 0.1)
-    args.attention_dropout = getattr(args, "attention_dropout", 0.1)
-    args.act_dropout = getattr(args, "act_dropout", 0.0)
-
-    args.encoder_ffn_embed_dim = getattr(args, "encoder_ffn_embed_dim", 4096)
-    args.encoder_layers = getattr(args, "encoder_layers", 6)
-    args.encoder_attention_heads = getattr(args, "encoder_attention_heads", 8)
-
-    args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 1024)
-    args.share_encoder_input_output_embed = getattr(
-        args, "share_encoder_input_output_embed", False
-    )
-    args.encoder_learned_pos = getattr(args, "encoder_learned_pos", False)
-    args.no_token_positional_embeddings = getattr(
-        args, "no_token_positional_embeddings", False
-    )
-    args.num_segment = getattr(args, "num_segment", 2)
-
-    args.sentence_class_num = getattr(args, "sentence_class_num", 2)
-    args.sent_loss = getattr(args, "sent_loss", False)
-
-    args.apply_bert_init = getattr(args, "apply_bert_init", False)
-
-    args.activation_fn = getattr(args, "activation_fn", "relu")
-    args.pooler_activation_fn = getattr(args, "pooler_activation_fn", "tanh")
-    args.encoder_normalize_before = getattr(args, "encoder_normalize_before", False)
-
-
-@register_model_architecture("masked_lm", "bert_base")
-def bert_base_architecture(args):
-    args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 768)
-    args.share_encoder_input_output_embed = getattr(
-        args, "share_encoder_input_output_embed", True
-    )
-    args.no_token_positional_embeddings = getattr(
-        args, "no_token_positional_embeddings", False
-    )
-    args.encoder_learned_pos = getattr(args, "encoder_learned_pos", True)
-    args.num_segment = getattr(args, "num_segment", 2)
-
-    args.encoder_layers = getattr(args, "encoder_layers", 12)
-
-    args.encoder_attention_heads = getattr(args, "encoder_attention_heads", 12)
-    args.encoder_ffn_embed_dim = getattr(args, "encoder_ffn_embed_dim", 3072)
-
-    args.sentence_class_num = getattr(args, "sentence_class_num", 2)
-    args.sent_loss = getattr(args, "sent_loss", True)
-
-    args.apply_bert_init = getattr(args, "apply_bert_init", True)
-
-    args.activation_fn = getattr(args, "activation_fn", "gelu")
-    args.pooler_activation_fn = getattr(args, "pooler_activation_fn", "tanh")
-    args.encoder_normalize_before = getattr(args, "encoder_normalize_before", True)
-    base_architecture(args)
-
-
-@register_model_architecture("masked_lm", "bert_large")
-def bert_large_architecture(args):
-    args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 1024)
-    args.encoder_layers = getattr(args, "encoder_layers", 24)
-    args.encoder_attention_heads = getattr(args, "encoder_attention_heads", 16)
-    args.encoder_ffn_embed_dim = getattr(args, "encoder_ffn_embed_dim", 4096)
-    bert_base_architecture(args)
-
-
-@register_model_architecture("masked_lm", "xlm_base")
-def xlm_architecture(args):
-    args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 1024)
-    args.share_encoder_input_output_embed = getattr(
-        args, "share_encoder_input_output_embed", True
-    )
-    args.no_token_positional_embeddings = getattr(
-        args, "no_token_positional_embeddings", False
-    )
-    args.encoder_learned_pos = getattr(args, "encoder_learned_pos", True)
-    args.num_segment = getattr(args, "num_segment", 1)
-
-    args.encoder_layers = getattr(args, "encoder_layers", 6)
-
-    args.encoder_attention_heads = getattr(args, "encoder_attention_heads", 8)
-    args.encoder_ffn_embed_dim = getattr(args, "encoder_ffn_embed_dim", 4096)
-
-    args.sent_loss = getattr(args, "sent_loss", False)
-
-    args.activation_fn = getattr(args, "activation_fn", "gelu")
-    args.encoder_normalize_before = getattr(args, "encoder_normalize_before", False)
-    args.pooler_activation_fn = getattr(args, "pooler_activation_fn", "tanh")
-    args.apply_bert_init = getattr(args, "apply_bert_init", True)
-    base_architecture(args)

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/model_utils.py

@@ -1,92 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-from typing import List, Optional
-
-import torch
-from torch import Tensor
-
-
-@torch.jit.script
-def script_skip_tensor_list(x: List[Tensor], mask):
-    res = [xi[mask] if xi.size(0) == mask.size(0) else xi[:, mask] for xi in x]
-    outputs = []
-    for i, t in enumerate(res):
-        if t.numel() != 0:
-            outputs.append(t)
-        else:
-            outputs.append(x[i])
-    return outputs
-
-
-@torch.jit.script
-def script_skip_tensor(x: Tensor, mask):
-    # None case
-    if x.size(0) == 0:
-        return x
-    res = x[mask] if x.size(0) == mask.size(0) else x[:, mask]
-    if res.numel() == 0:
-        return x
-    else:
-        return res
-
-
-@torch.jit.script
-def expand_2d_or_3d_tensor(x, trg_dim: int, padding_idx: int):
-    """
-    Expand 2D/3D tensor on dim=1
-    """
-    if x is None:
-        return None
-
-    assert x.dim() == 2 or x.dim() == 3
-    assert trg_dim >= x.size(1), (trg_dim, x.size())
-    if trg_dim == x.size(1):
-        return x
-
-    dims = [x.size(0), trg_dim - x.size(1)]
-    if x.dim() == 3:
-        dims.append(x.size(2))
-    x = torch.cat([x, torch.zeros(dims).to(x).fill_(padding_idx)], 1)
-
-    return x
-
-
-@torch.jit.script
-def coalesce(x: Optional[Tensor], y: Tensor) -> Tensor:
-    return x if x is not None else y
-
-
-@torch.jit.script
-def fill_tensors(
-    x: Optional[Tensor], mask, y: Optional[Tensor], padding_idx: int
-) -> Optional[Tensor]:
-    """
-    Filling tensor x with y at masked positions (dim=0).
-    """
-    if x is None or x.size()[0] == 0 or y is None:
-        return x
-    assert x.dim() == y.dim() and mask.size(0) == x.size(0)
-    assert x.dim() == 2 or (x.dim() == 3 and x.size(2) == y.size(2))
-
-    n_selected = mask.sum()
-    if n_selected == 0:
-        return x
-    assert n_selected == y.size(0)
-    if n_selected == x.size(0):
-        return y
-
-    if x.size(1) < y.size(1):
-        x = expand_2d_or_3d_tensor(x, y.size(1), padding_idx)
-        x[mask] = y
-    elif x.size(1) > y.size(1):
-        x[mask] = torch.tensor(padding_idx).type_as(x)
-        if x.dim() == 2:
-            x[mask, : y.size(1)] = y
-        else:
-            x[mask, : y.size(1), :] = y
-    else:
-        x[mask] = y
-    return x

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/multilingual_transformer.py

@@ -1,229 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-from collections import OrderedDict
-
-from fairseq import utils
-from fairseq.models import (
-    FairseqMultiModel,
-    register_model,
-    register_model_architecture,
-)
-from fairseq.models.transformer import (
-    Embedding,
-    TransformerDecoder,
-    TransformerEncoder,
-    TransformerModel,
-    base_architecture,
-)
-from fairseq.utils import safe_hasattr
-
-
-@register_model("multilingual_transformer")
-class MultilingualTransformerModel(FairseqMultiModel):
-    """Train Transformer models for multiple language pairs simultaneously.
-
-    Requires `--task multilingual_translation`.
-
-    We inherit all arguments from TransformerModel and assume that all language
-    pairs use a single Transformer architecture. In addition, we provide several
-    options that are specific to the multilingual setting.
-
-    Args:
-        --share-encoder-embeddings: share encoder embeddings across all source languages
-        --share-decoder-embeddings: share decoder embeddings across all target languages
-        --share-encoders: share all encoder params (incl. embeddings) across all source languages
-        --share-decoders: share all decoder params (incl. embeddings) across all target languages
-    """
-
-    def __init__(self, encoders, decoders):
-        super().__init__(encoders, decoders)
-
-    @staticmethod
-    def add_args(parser):
-        """Add model-specific arguments to the parser."""
-        TransformerModel.add_args(parser)
-        parser.add_argument(
-            "--share-encoder-embeddings",
-            action="store_true",
-            help="share encoder embeddings across languages",
-        )
-        parser.add_argument(
-            "--share-decoder-embeddings",
-            action="store_true",
-            help="share decoder embeddings across languages",
-        )
-        parser.add_argument(
-            "--share-encoders",
-            action="store_true",
-            help="share encoders across languages",
-        )
-        parser.add_argument(
-            "--share-decoders",
-            action="store_true",
-            help="share decoders across languages",
-        )
-
-    @classmethod
-    def build_model(cls, args, task):
-        """Build a new model instance."""
-        from fairseq.tasks.multilingual_translation import MultilingualTranslationTask
-
-        assert isinstance(task, MultilingualTranslationTask)
-
-        # make sure all arguments are present in older models
-        base_multilingual_architecture(args)
-
-        if not safe_hasattr(args, "max_source_positions"):
-            args.max_source_positions = 1024
-        if not safe_hasattr(args, "max_target_positions"):
-            args.max_target_positions = 1024
-
-        src_langs = [lang_pair.split("-")[0] for lang_pair in task.model_lang_pairs]
-        tgt_langs = [lang_pair.split("-")[1] for lang_pair in task.model_lang_pairs]
-
-        if args.share_encoders:
-            args.share_encoder_embeddings = True
-        if args.share_decoders:
-            args.share_decoder_embeddings = True
-
-        def build_embedding(dictionary, embed_dim, path=None):
-            num_embeddings = len(dictionary)
-            padding_idx = dictionary.pad()
-            emb = Embedding(num_embeddings, embed_dim, padding_idx)
-            # if provided, load from preloaded dictionaries
-            if path:
-                embed_dict = utils.parse_embedding(path)
-                utils.load_embedding(embed_dict, dictionary, emb)
-            return emb
-
-        # build shared embeddings (if applicable)
-        shared_encoder_embed_tokens, shared_decoder_embed_tokens = None, None
-        if args.share_all_embeddings:
-            if args.encoder_embed_dim != args.decoder_embed_dim:
-                raise ValueError(
-                    "--share-all-embeddings requires --encoder-embed-dim to match --decoder-embed-dim"
-                )
-            if args.decoder_embed_path and (
-                args.decoder_embed_path != args.encoder_embed_path
-            ):
-                raise ValueError(
-                    "--share-all-embeddings not compatible with --decoder-embed-path"
-                )
-            shared_encoder_embed_tokens = FairseqMultiModel.build_shared_embeddings(
-                dicts=task.dicts,
-                langs=task.langs,
-                embed_dim=args.encoder_embed_dim,
-                build_embedding=build_embedding,
-                pretrained_embed_path=args.encoder_embed_path,
-            )
-            shared_decoder_embed_tokens = shared_encoder_embed_tokens
-            args.share_decoder_input_output_embed = True
-        else:
-            if args.share_encoder_embeddings:
-                shared_encoder_embed_tokens = FairseqMultiModel.build_shared_embeddings(
-                    dicts=task.dicts,
-                    langs=src_langs,
-                    embed_dim=args.encoder_embed_dim,
-                    build_embedding=build_embedding,
-                    pretrained_embed_path=args.encoder_embed_path,
-                )
-            if args.share_decoder_embeddings:
-                shared_decoder_embed_tokens = FairseqMultiModel.build_shared_embeddings(
-                    dicts=task.dicts,
-                    langs=tgt_langs,
-                    embed_dim=args.decoder_embed_dim,
-                    build_embedding=build_embedding,
-                    pretrained_embed_path=args.decoder_embed_path,
-                )
-
-        # encoders/decoders for each language
-        lang_encoders, lang_decoders = {}, {}
-
-        def get_encoder(lang):
-            if lang not in lang_encoders:
-                if shared_encoder_embed_tokens is not None:
-                    encoder_embed_tokens = shared_encoder_embed_tokens
-                else:
-                    encoder_embed_tokens = build_embedding(
-                        task.dicts[lang],
-                        args.encoder_embed_dim,
-                        args.encoder_embed_path,
-                    )
-                lang_encoders[lang] = cls._get_module_class(
-                    True, args, task.dicts[lang], encoder_embed_tokens, src_langs
-                )
-            return lang_encoders[lang]
-
-        def get_decoder(lang):
-            if lang not in lang_decoders:
-                if shared_decoder_embed_tokens is not None:
-                    decoder_embed_tokens = shared_decoder_embed_tokens
-                else:
-                    decoder_embed_tokens = build_embedding(
-                        task.dicts[lang],
-                        args.decoder_embed_dim,
-                        args.decoder_embed_path,
-                    )
-                lang_decoders[lang] = cls._get_module_class(
-                    False, args, task.dicts[lang], decoder_embed_tokens, tgt_langs
-                )
-            return lang_decoders[lang]
-
-        # shared encoders/decoders (if applicable)
-        shared_encoder, shared_decoder = None, None
-        if args.share_encoders:
-            shared_encoder = get_encoder(src_langs[0])
-        if args.share_decoders:
-            shared_decoder = get_decoder(tgt_langs[0])
-
-        encoders, decoders = OrderedDict(), OrderedDict()
-        for lang_pair, src, tgt in zip(task.model_lang_pairs, src_langs, tgt_langs):
-            encoders[lang_pair] = (
-                shared_encoder if shared_encoder is not None else get_encoder(src)
-            )
-            decoders[lang_pair] = (
-                shared_decoder if shared_decoder is not None else get_decoder(tgt)
-            )
-
-        return MultilingualTransformerModel(encoders, decoders)
-
-    @classmethod
-    def _get_module_class(cls, is_encoder, args, lang_dict, embed_tokens, langs):
-        module_class = TransformerEncoder if is_encoder else TransformerDecoder
-        return module_class(args, lang_dict, embed_tokens)
-
-    def load_state_dict(self, state_dict, strict=True, model_cfg=None):
-        state_dict_subset = state_dict.copy()
-        for k, _ in state_dict.items():
-            assert k.startswith("models.")
-            lang_pair = k.split(".")[1]
-            if lang_pair not in self.models:
-                del state_dict_subset[k]
-        super().load_state_dict(state_dict_subset, strict=strict, model_cfg=model_cfg)
-
-
-@register_model_architecture("multilingual_transformer", "multilingual_transformer")
-def base_multilingual_architecture(args):
-    base_architecture(args)
-    args.share_encoder_embeddings = getattr(args, "share_encoder_embeddings", False)
-    args.share_decoder_embeddings = getattr(args, "share_decoder_embeddings", False)
-    args.share_encoders = getattr(args, "share_encoders", False)
-    args.share_decoders = getattr(args, "share_decoders", False)
-
-
-@register_model_architecture(
-    "multilingual_transformer", "multilingual_transformer_iwslt_de_en"
-)
-def multilingual_transformer_iwslt_de_en(args):
-    args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 512)
-    args.encoder_ffn_embed_dim = getattr(args, "encoder_ffn_embed_dim", 1024)
-    args.encoder_attention_heads = getattr(args, "encoder_attention_heads", 4)
-    args.encoder_layers = getattr(args, "encoder_layers", 6)
-    args.decoder_embed_dim = getattr(args, "decoder_embed_dim", 512)
-    args.decoder_ffn_embed_dim = getattr(args, "decoder_ffn_embed_dim", 1024)
-    args.decoder_attention_heads = getattr(args, "decoder_attention_heads", 4)
-    args.decoder_layers = getattr(args, "decoder_layers", 6)
-    base_multilingual_architecture(args)

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/nat/__init__.py

@@ -1,13 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-"""isort:skip_file"""
-
-from .fairseq_nat_model import *
-from .nonautoregressive_transformer import *
-from .nat_crf_transformer import *
-from .iterative_nonautoregressive_transformer import *
-from .cmlm_transformer import *
-from .levenshtein_transformer import *
-from .insertion_transformer import *

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/nat/cmlm_transformer.py

@@ -1,162 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-"""
-This file implements:
-Ghazvininejad, Marjan, et al.
-"Constant-time machine translation with conditional masked language models."
-arXiv preprint arXiv:1904.09324 (2019).
-"""
-
-from fairseq.models import register_model, register_model_architecture
-from fairseq.models.nat import NATransformerModel
-from fairseq.utils import new_arange
-
-
-def _skeptical_unmasking(output_scores, output_masks, p):
-    sorted_index = output_scores.sort(-1)[1]
-    boundary_len = (
-        (output_masks.sum(1, keepdim=True).type_as(output_scores) - 2) * p
-    ).long()
-    skeptical_mask = new_arange(output_masks) < boundary_len
-    return skeptical_mask.scatter(1, sorted_index, skeptical_mask)
-
-
-@register_model("cmlm_transformer")
-class CMLMNATransformerModel(NATransformerModel):
-    @staticmethod
-    def add_args(parser):
-        NATransformerModel.add_args(parser)
-
-    def forward(
-        self, src_tokens, src_lengths, prev_output_tokens, tgt_tokens, **kwargs
-    ):
-        assert not self.decoder.src_embedding_copy, "do not support embedding copy."
-
-        # encoding
-        encoder_out = self.encoder(src_tokens, src_lengths=src_lengths, **kwargs)
-        # length prediction
-        length_out = self.decoder.forward_length(
-            normalize=False, encoder_out=encoder_out
-        )
-        length_tgt = self.decoder.forward_length_prediction(
-            length_out, encoder_out, tgt_tokens
-        )
-
-        # decoding
-        word_ins_out = self.decoder(
-            normalize=False,
-            prev_output_tokens=prev_output_tokens,
-            encoder_out=encoder_out,
-        )
-        word_ins_mask = prev_output_tokens.eq(self.unk)
-
-        return {
-            "word_ins": {
-                "out": word_ins_out,
-                "tgt": tgt_tokens,
-                "mask": word_ins_mask,
-                "ls": self.args.label_smoothing,
-                "nll_loss": True,
-            },
-            "length": {
-                "out": length_out,
-                "tgt": length_tgt,
-                "factor": self.decoder.length_loss_factor,
-            },
-        }
-
-    def forward_decoder(self, decoder_out, encoder_out, decoding_format=None, **kwargs):
-
-        step = decoder_out.step
-        max_step = decoder_out.max_step
-
-        output_tokens = decoder_out.output_tokens
-        output_scores = decoder_out.output_scores
-        history = decoder_out.history
-
-        # execute the decoder
-        output_masks = output_tokens.eq(self.unk)
-        _scores, _tokens = self.decoder(
-            normalize=True,
-            prev_output_tokens=output_tokens,
-            encoder_out=encoder_out,
-        ).max(-1)
-        output_tokens.masked_scatter_(output_masks, _tokens[output_masks])
-        output_scores.masked_scatter_(output_masks, _scores[output_masks])
-
-        if history is not None:
-            history.append(output_tokens.clone())
-
-        # skeptical decoding (depend on the maximum decoding steps.)
-        if (step + 1) < max_step:
-            skeptical_mask = _skeptical_unmasking(
-                output_scores, output_tokens.ne(self.pad), 1 - (step + 1) / max_step
-            )
-
-            output_tokens.masked_fill_(skeptical_mask, self.unk)
-            output_scores.masked_fill_(skeptical_mask, 0.0)
-
-            if history is not None:
-                history.append(output_tokens.clone())
-
-        return decoder_out._replace(
-            output_tokens=output_tokens,
-            output_scores=output_scores,
-            attn=None,
-            history=history,
-        )
-
-
-@register_model_architecture("cmlm_transformer", "cmlm_transformer")
-def cmlm_base_architecture(args):
-    args.encoder_embed_path = getattr(args, "encoder_embed_path", None)
-    args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 512)
-    args.encoder_ffn_embed_dim = getattr(args, "encoder_ffn_embed_dim", 2048)
-    args.encoder_layers = getattr(args, "encoder_layers", 6)
-    args.encoder_attention_heads = getattr(args, "encoder_attention_heads", 8)
-    args.encoder_normalize_before = getattr(args, "encoder_normalize_before", False)
-    args.encoder_learned_pos = getattr(args, "encoder_learned_pos", False)
-    args.decoder_embed_path = getattr(args, "decoder_embed_path", None)
-    args.decoder_embed_dim = getattr(args, "decoder_embed_dim", args.encoder_embed_dim)
-    args.decoder_ffn_embed_dim = getattr(
-        args, "decoder_ffn_embed_dim", args.encoder_ffn_embed_dim
-    )
-    args.decoder_layers = getattr(args, "decoder_layers", 6)
-    args.decoder_attention_heads = getattr(args, "decoder_attention_heads", 8)
-    args.decoder_normalize_before = getattr(args, "decoder_normalize_before", False)
-    args.decoder_learned_pos = getattr(args, "decoder_learned_pos", False)
-    args.attention_dropout = getattr(args, "attention_dropout", 0.0)
-    args.activation_dropout = getattr(args, "activation_dropout", 0.0)
-    args.activation_fn = getattr(args, "activation_fn", "relu")
-    args.dropout = getattr(args, "dropout", 0.1)
-    args.adaptive_softmax_cutoff = getattr(args, "adaptive_softmax_cutoff", None)
-    args.adaptive_softmax_dropout = getattr(args, "adaptive_softmax_dropout", 0)
-    args.share_decoder_input_output_embed = getattr(
-        args, "share_decoder_input_output_embed", False
-    )
-    args.share_all_embeddings = getattr(args, "share_all_embeddings", True)
-    args.no_token_positional_embeddings = getattr(
-        args, "no_token_positional_embeddings", False
-    )
-    args.adaptive_input = getattr(args, "adaptive_input", False)
-    args.apply_bert_init = getattr(args, "apply_bert_init", False)
-
-    args.decoder_output_dim = getattr(
-        args, "decoder_output_dim", args.decoder_embed_dim
-    )
-    args.decoder_input_dim = getattr(args, "decoder_input_dim", args.decoder_embed_dim)
-
-    # --- special arguments ---
-    args.sg_length_pred = getattr(args, "sg_length_pred", False)
-    args.pred_length_offset = getattr(args, "pred_length_offset", False)
-    args.length_loss_factor = getattr(args, "length_loss_factor", 0.1)
-    args.ngram_predictor = getattr(args, "ngram_predictor", 1)
-    args.src_embedding_copy = getattr(args, "src_embedding_copy", False)
-
-
-@register_model_architecture("cmlm_transformer", "cmlm_transformer_wmt_en_de")
-def cmlm_wmt_en_de(args):
-    cmlm_base_architecture(args)

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/nat/fairseq_nat_model.py

@@ -1,172 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-import math
-
-import torch
-from fairseq.models.transformer import (
-    TransformerDecoder,
-    TransformerEncoder,
-    TransformerModel,
-)
-from fairseq.modules.transformer_sentence_encoder import init_bert_params
-
-
-def ensemble_encoder(func):
-    def wrapper(self, *args, **kwargs):
-        if self.ensemble_models is None or len(self.ensemble_models) == 1:
-            return func(self, *args, **kwargs)
-        encoder_outs = [
-            func(model, *args, **kwargs, return_all_hiddens=True)
-            for model in self.ensemble_models
-        ]
-        _encoder_out = encoder_outs[0].copy()
-
-        def stack(key):
-            outs = [e[key][0] for e in encoder_outs]
-            return [torch.stack(outs, -1) if outs[0] is not None else None]
-
-        _encoder_out["encoder_out"] = stack("encoder_out")
-        _encoder_out["encoder_embedding"] = stack("encoder_embedding")
-
-        num_layers = len(_encoder_out["encoder_states"])
-        if num_layers > 0:
-            _encoder_out["encoder_states"] = [
-                torch.stack([e["encoder_states"][i] for e in encoder_outs], -1)
-                for i in range(num_layers)
-            ]
-        return _encoder_out
-
-    return wrapper
-
-
-def ensemble_decoder(func):
-    def wrapper(self, normalize=False, encoder_out=None, *args, **kwargs):
-        if self.ensemble_models is None or len(self.ensemble_models) == 1:
-            return func(
-                self, normalize=normalize, encoder_out=encoder_out, *args, **kwargs
-            )
-
-        def _replace(encoder_out, new_val):
-            new_encoder_out = encoder_out.copy()
-            new_encoder_out["encoder_out"] = [new_val]
-            return new_encoder_out
-
-        action_outs = [
-            func(
-                model,
-                normalize=normalize,
-                encoder_out=_replace(
-                    encoder_out, encoder_out["encoder_out"][0][:, :, :, i]
-                ),
-                *args,
-                **kwargs
-            )
-            for i, model in enumerate(self.ensemble_models)
-        ]
-
-        if not isinstance(action_outs[0], tuple):  # return multiple values
-            action_outs = [[a] for a in action_outs]
-        else:
-            action_outs = [list(a) for a in action_outs]
-
-        ensembled_outs = []
-        for i in range(len(action_outs[0])):
-            if i == 0 and normalize:
-                ensembled_outs += [
-                    torch.logsumexp(
-                        torch.stack([a[i] for a in action_outs], -1), dim=-1
-                    )
-                    - math.log(len(self.ensemble_models))
-                ]
-            elif action_outs[0][i] is not None:
-                ensembled_outs += [torch.stack([a[i] for a in action_outs], -1)]
-            else:
-                ensembled_outs += [None]
-
-        if len(ensembled_outs) == 1:
-            return ensembled_outs[0]
-        return tuple(ensembled_outs)
-
-    return wrapper
-
-
-class FairseqNATModel(TransformerModel):
-    """
-    Abstract class for all nonautoregressive-based models
-    """
-
-    def __init__(self, args, encoder, decoder):
-        super().__init__(args, encoder, decoder)
-        self.tgt_dict = decoder.dictionary
-        self.bos = decoder.dictionary.bos()
-        self.eos = decoder.dictionary.eos()
-        self.pad = decoder.dictionary.pad()
-        self.unk = decoder.dictionary.unk()
-
-        self.ensemble_models = None
-
-    @property
-    def allow_length_beam(self):
-        return False
-
-    @property
-    def allow_ensemble(self):
-        return True
-
-    def enable_ensemble(self, models):
-        self.encoder.ensemble_models = [m.encoder for m in models]
-        self.decoder.ensemble_models = [m.decoder for m in models]
-
-    @staticmethod
-    def add_args(parser):
-        TransformerModel.add_args(parser)
-        parser.add_argument(
-            "--apply-bert-init",
-            action="store_true",
-            help="use custom param initialization for BERT",
-        )
-
-    @classmethod
-    def build_decoder(cls, args, tgt_dict, embed_tokens):
-        decoder = FairseqNATDecoder(args, tgt_dict, embed_tokens)
-        if getattr(args, "apply_bert_init", False):
-            decoder.apply(init_bert_params)
-        return decoder
-
-    @classmethod
-    def build_encoder(cls, args, src_dict, embed_tokens):
-        encoder = FairseqNATEncoder(args, src_dict, embed_tokens)
-        if getattr(args, "apply_bert_init", False):
-            encoder.apply(init_bert_params)
-        return encoder
-
-    def forward_encoder(self, encoder_inputs):
-        return self.encoder(*encoder_inputs)
-
-    def forward_decoder(self, *args, **kwargs):
-        return NotImplementedError
-
-    def initialize_output_tokens(self, *args, **kwargs):
-        return NotImplementedError
-
-    def forward(self, *args, **kwargs):
-        return NotImplementedError
-
-
-class FairseqNATEncoder(TransformerEncoder):
-    def __init__(self, args, dictionary, embed_tokens):
-        super().__init__(args, dictionary, embed_tokens)
-        self.ensemble_models = None
-
-    @ensemble_encoder
-    def forward(self, *args, **kwargs):
-        return super().forward(*args, **kwargs)
-
-
-class FairseqNATDecoder(TransformerDecoder):
-    def __init__(self, args, dictionary, embed_tokens, no_encoder_attn=False):
-        super().__init__(args, dictionary, embed_tokens, no_encoder_attn)
-        self.ensemble_models = None

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/nat/insertion_transformer.py

@@ -1,280 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-import numpy as np
-import torch
-import torch.nn.functional as F
-from fairseq.models import register_model, register_model_architecture
-from fairseq.models.nat import (
-    FairseqNATModel,
-    LevenshteinTransformerDecoder,
-    LevenshteinTransformerModel,
-    ensemble_decoder,
-)
-from fairseq.models.transformer import Linear
-from fairseq.modules.transformer_sentence_encoder import init_bert_params
-from fairseq.utils import new_arange
-
-
-class NegativeDistanceScore(object):
-    def __init__(self):
-
-        # pre-compute some values
-        self.scores = {}
-
-        self.scores[0.5] = self.compute_score_full(50, 0.5)
-        self.scores[1.0] = self.compute_score_full(50, 1.0)
-        self.scores[2.0] = self.compute_score_full(50, 2.0)
-
-    def __call__(self, i, L, tau):
-        if (tau is None) or (tau > 1000):
-            return 1 / L
-
-        if tau in self.scores:
-            if L < self.scores[tau].shape[0]:
-                return self.scores[tau][L - 1, i]
-        return self.compute_score(L, tau)[i]
-
-    def compute_score(self, L, tau):
-        s = np.array([-abs(L / 2 - i) / tau for i in range(L)])
-        s = np.exp(s - s.max())
-        return s / s.sum()
-
-    def compute_score_full(self, L, tau):
-        s = -abs(np.arange(0, L - 1)[:, None] / 2 - np.arange(L)[None, :]) / tau
-        s = np.tril(s, 0) + np.triu(s - float("inf"), 1)
-        s = np.exp(s - s.max(1, keepdims=True))
-        return s / s.sum(1, keepdims=True)
-
-
-neg_scorer = NegativeDistanceScore()
-
-
-def _get_ins_targets(in_tokens, out_tokens, padding_idx, unk_idx, vocab_size, tau=None):
-    try:
-        from fairseq import libnat
-    except ImportError as e:
-        import sys
-
-        sys.stderr.write("ERROR: missing libnat. run `pip install --editable .`\n")
-        raise e
-
-    B = in_tokens.size(0)
-    T = in_tokens.size(1)
-    V = vocab_size
-
-    with torch.cuda.device_of(in_tokens):
-        in_tokens_list = [
-            [t for t in s if t != padding_idx] for i, s in enumerate(in_tokens.tolist())
-        ]
-        out_tokens_list = [
-            [t for t in s if t != padding_idx]
-            for i, s in enumerate(out_tokens.tolist())
-        ]
-
-    full_labels = libnat.suggested_ed2_path(
-        in_tokens_list, out_tokens_list, padding_idx
-    )
-    insert_labels = [a[:-1] for a in full_labels]
-
-    # numericalize1
-    insert_label_tensors = in_tokens.new_zeros(B * (T - 1) * V).float()
-    insert_index, insert_labels = zip(
-        *[
-            (w + (j + i * (T - 1)) * V, neg_scorer(k, len(label), tau))
-            for i, labels in enumerate(insert_labels)
-            for j, label in enumerate(labels[1:-1])
-            for k, w in enumerate(label)
-        ]
-    )  # HACK 1:-1
-    insert_index, insert_labels = [
-        torch.tensor(list(a), device=in_tokens.device)
-        for a in [insert_index, insert_labels]
-    ]
-    insert_label_tensors.scatter_(0, insert_index.long(), insert_labels)
-    insert_label_tensors = insert_label_tensors.view(B, T - 1, V)
-
-    return insert_label_tensors
-
-
-def _apply_ins_words(in_tokens, in_scores, word_ins_pred, word_ins_scores, padding_idx):
-
-    padding_masks = in_tokens[:, 1:].eq(padding_idx)
-    word_ins_scores.masked_fill_(padding_masks, 0.0)
-    word_ins_pred.masked_fill_(padding_masks, padding_idx)
-
-    in_coords = new_arange(in_tokens).type_as(in_scores)
-
-    # shift all padding predictions to infinite
-    out_coords = (in_coords[:, 1:] - 0.5).masked_fill(
-        word_ins_pred.eq(padding_idx), float("inf")
-    )
-    out_coords = torch.cat([in_coords, out_coords], 1).sort(-1)[1]
-    out_tokens = torch.cat([in_tokens, word_ins_pred], 1).gather(1, out_coords)
-    out_scores = torch.cat([in_scores, word_ins_scores], 1).gather(1, out_coords)
-    return out_tokens, out_scores
-
-
-@register_model("insertion_transformer")
-class InsertionTransformerModel(LevenshteinTransformerModel):
-    def __init__(self, args, encoder, decoder):
-        super().__init__(args, encoder, decoder)
-
-    @staticmethod
-    def add_args(parser):
-        FairseqNATModel.add_args(parser)
-        parser.add_argument("--label-tau", default=None, type=float)
-
-    @classmethod
-    def build_decoder(cls, args, tgt_dict, embed_tokens):
-        decoder = InsertionTransformerDecoder(args, tgt_dict, embed_tokens)
-        if getattr(args, "apply_bert_init", False):
-            decoder.apply(init_bert_params)
-        return decoder
-
-    def forward(
-        self, src_tokens, src_lengths, prev_output_tokens, tgt_tokens, **kwargs
-    ):
-
-        assert tgt_tokens is not None, "forward function only supports training."
-
-        # encoding
-        encoder_out = self.encoder(src_tokens, src_lengths=src_lengths, **kwargs)
-
-        # generate training labels for insertion
-        word_ins_out = self.decoder.forward_word_ins(
-            normalize=False,
-            prev_output_tokens=prev_output_tokens,
-            encoder_out=encoder_out,
-        )
-
-        word_ins_tgt = _get_ins_targets(
-            prev_output_tokens,
-            tgt_tokens,
-            self.pad,
-            self.unk,
-            len(self.tgt_dict),
-            tau=self.decoder.label_tau,
-        ).type_as(word_ins_out)
-        word_ins_masks = prev_output_tokens[:, 1:].ne(self.pad)
-
-        return {
-            "word_ins": {
-                "out": word_ins_out,
-                "tgt": word_ins_tgt,
-                "mask": word_ins_masks,
-                "ls": self.args.label_smoothing,
-                "nll_loss": True,
-            }
-        }
-
-    def forward_decoder(
-        self, decoder_out, encoder_out, eos_penalty=0.0, max_ratio=None, **kwargs
-    ):
-
-        output_tokens = decoder_out.output_tokens
-        output_scores = decoder_out.output_scores
-        history = decoder_out.history
-
-        # TODO: decoding for InsertionTransformer
-        word_ins_score = self.decoder.forward_word_ins(
-            normalize=True, prev_output_tokens=output_tokens, encoder_out=encoder_out
-        )
-
-        if eos_penalty > 0.0:
-            word_ins_score[:, :, self.pad] -= eos_penalty
-        word_ins_score, word_ins_pred = word_ins_score.max(-1)
-        output_tokens, output_scores = _apply_ins_words(
-            output_tokens, output_scores, word_ins_pred, word_ins_score, self.pad
-        )
-
-        # delete some unnecessary paddings
-        cut_off = output_tokens.ne(self.pad).sum(1).max()
-        output_tokens = output_tokens[:, :cut_off]
-        output_scores = output_scores[:, :cut_off]
-
-        if history is not None:
-            history.append(output_tokens.clone())
-
-        return decoder_out._replace(
-            output_tokens=output_tokens,
-            output_scores=output_scores,
-            attn=None,
-            history=history,
-        )
-
-
-class InsertionTransformerDecoder(LevenshteinTransformerDecoder):
-    def __init__(self, args, dictionary, embed_tokens, no_encoder_attn=False):
-        # use the TransformerDecoder's __init__
-        super(LevenshteinTransformerDecoder, self).__init__(
-            args, dictionary, embed_tokens, no_encoder_attn=no_encoder_attn
-        )
-
-        self.dictionary = dictionary
-        self.bos = dictionary.bos()
-        self.unk = dictionary.unk()
-        self.eos = dictionary.eos()
-        self.pool_out = Linear(self.output_embed_dim * 2, self.output_embed_dim)
-
-        self.label_tau = getattr(args, "label_tau", None)
-
-    @ensemble_decoder
-    def forward_word_ins(self, normalize, encoder_out, prev_output_tokens):
-        features = self.extract_features(prev_output_tokens, encoder_out=encoder_out)[0]
-        features = self.pool_out(
-            torch.cat([features[:, :-1, :], features[:, 1:, :]], 2)
-        )
-        decoder_out = self.output_layer(features)
-        return F.log_softmax(decoder_out, -1) if normalize else decoder_out
-
-    def forward_mask_ins(self, *args, **kwargs):
-        raise NotImplementedError
-
-    def forward_word_del(self, *args, **kwargs):
-        raise NotImplementedError
-
-
-@register_model_architecture("insertion_transformer", "insertion_transformer")
-def insertion_base_architecture(args):
-    args.encoder_embed_path = getattr(args, "encoder_embed_path", None)
-    args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 512)
-    args.encoder_ffn_embed_dim = getattr(args, "encoder_ffn_embed_dim", 2048)
-    args.encoder_layers = getattr(args, "encoder_layers", 6)
-    args.encoder_attention_heads = getattr(args, "encoder_attention_heads", 8)
-    args.encoder_normalize_before = getattr(args, "encoder_normalize_before", False)
-    args.encoder_learned_pos = getattr(args, "encoder_learned_pos", False)
-    args.decoder_embed_path = getattr(args, "decoder_embed_path", None)
-    args.decoder_embed_dim = getattr(args, "decoder_embed_dim", args.encoder_embed_dim)
-    args.decoder_ffn_embed_dim = getattr(
-        args, "decoder_ffn_embed_dim", args.encoder_ffn_embed_dim
-    )
-    args.decoder_layers = getattr(args, "decoder_layers", 6)
-    args.decoder_attention_heads = getattr(args, "decoder_attention_heads", 8)
-    args.decoder_normalize_before = getattr(args, "decoder_normalize_before", False)
-    args.decoder_learned_pos = getattr(args, "decoder_learned_pos", False)
-    args.attention_dropout = getattr(args, "attention_dropout", 0.0)
-    args.activation_dropout = getattr(args, "activation_dropout", 0.0)
-    args.activation_fn = getattr(args, "activation_fn", "relu")
-    args.dropout = getattr(args, "dropout", 0.1)
-    args.adaptive_softmax_cutoff = getattr(args, "adaptive_softmax_cutoff", None)
-    args.adaptive_softmax_dropout = getattr(args, "adaptive_softmax_dropout", 0)
-    args.share_decoder_input_output_embed = getattr(
-        args, "share_decoder_input_output_embed", False
-    )
-    args.share_all_embeddings = getattr(args, "share_all_embeddings", False)
-    args.no_token_positional_embeddings = getattr(
-        args, "no_token_positional_embeddings", False
-    )
-    args.adaptive_input = getattr(args, "adaptive_input", False)
-    args.apply_bert_init = getattr(args, "apply_bert_init", False)
-
-    args.decoder_output_dim = getattr(
-        args, "decoder_output_dim", args.decoder_embed_dim
-    )
-    args.decoder_input_dim = getattr(args, "decoder_input_dim", args.decoder_embed_dim)
-
-    # special for insertion transformer
-    args.label_tau = getattr(args, "label_tau", None)

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/nat/iterative_nonautoregressive_transformer.py

@@ -1,228 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-import torch
-from fairseq.models import register_model, register_model_architecture
-from fairseq.models.nat import NATransformerModel
-
-
-def _sequential_poisoning(s, V, beta=0.33, bos=2, eos=3, pad=1):
-    # s: input batch
-    # V: vocabulary size
-    rand_words = torch.randint(low=4, high=V, size=s.size(), device=s.device)
-    choices = torch.rand(size=s.size(), device=s.device)
-    choices.masked_fill_((s == pad) | (s == bos) | (s == eos), 1)
-
-    replace = choices < beta / 3
-    repeat = (choices >= beta / 3) & (choices < beta * 2 / 3)
-    swap = (choices >= beta * 2 / 3) & (choices < beta)
-    safe = choices >= beta
-
-    for i in range(s.size(1) - 1):
-        rand_word = rand_words[:, i]
-        next_word = s[:, i + 1]
-        self_word = s[:, i]
-
-        replace_i = replace[:, i]
-        swap_i = swap[:, i] & (next_word != 3)
-        repeat_i = repeat[:, i] & (next_word != 3)
-        safe_i = safe[:, i] | ((next_word == 3) & (~replace_i))
-
-        s[:, i] = (
-            self_word * (safe_i | repeat_i).long()
-            + next_word * swap_i.long()
-            + rand_word * replace_i.long()
-        )
-        s[:, i + 1] = (
-            next_word * (safe_i | replace_i).long()
-            + self_word * (swap_i | repeat_i).long()
-        )
-    return s
-
-
-def gumbel_noise(input, TINY=1e-8):
-    return (
-        input.new_zeros(*input.size())
-        .uniform_()
-        .add_(TINY)
-        .log_()
-        .neg_()
-        .add_(TINY)
-        .log_()
-        .neg_()
-    )
-
-
-@register_model("iterative_nonautoregressive_transformer")
-class IterNATransformerModel(NATransformerModel):
-    @staticmethod
-    def add_args(parser):
-        NATransformerModel.add_args(parser)
-        parser.add_argument(
-            "--train-step",
-            type=int,
-            help="number of refinement iterations during training",
-        )
-        parser.add_argument(
-            "--dae-ratio",
-            type=float,
-            help="the probability of switching to the denoising auto-encoder loss",
-        )
-        parser.add_argument(
-            "--stochastic-approx",
-            action="store_true",
-            help="sampling from the decoder as the inputs for next iteration",
-        )
-
-    @classmethod
-    def build_model(cls, args, task):
-        model = super().build_model(args, task)
-        model.train_step = getattr(args, "train_step", 4)
-        model.dae_ratio = getattr(args, "dae_ratio", 0.5)
-        model.stochastic_approx = getattr(args, "stochastic_approx", False)
-        return model
-
-    def forward(
-        self, src_tokens, src_lengths, prev_output_tokens, tgt_tokens, **kwargs
-    ):
-
-        B, T = prev_output_tokens.size()
-
-        # encoding
-        encoder_out = self.encoder(src_tokens, src_lengths=src_lengths, **kwargs)
-
-        # length prediction
-        length_out = self.decoder.forward_length(
-            normalize=False, encoder_out=encoder_out
-        )
-        length_tgt = self.decoder.forward_length_prediction(
-            length_out, encoder_out, tgt_tokens
-        )
-
-        # decoding
-        word_ins_outs, word_ins_tgts, word_ins_masks = [], [], []
-        for t in range(self.train_step):
-            word_ins_out = self.decoder(
-                normalize=False,
-                prev_output_tokens=prev_output_tokens,
-                encoder_out=encoder_out,
-                step=t,
-            )
-            word_ins_tgt = tgt_tokens
-            word_ins_mask = word_ins_tgt.ne(self.pad)
-
-            word_ins_outs.append(word_ins_out)
-            word_ins_tgts.append(word_ins_tgt)
-            word_ins_masks.append(word_ins_mask)
-
-            if t < (self.train_step - 1):
-                # prediction for next iteration
-                if self.stochastic_approx:
-                    word_ins_prediction = (
-                        word_ins_out + gumbel_noise(word_ins_out)
-                    ).max(-1)[1]
-                else:
-                    word_ins_prediction = word_ins_out.max(-1)[1]
-
-                prev_output_tokens = prev_output_tokens.masked_scatter(
-                    word_ins_mask, word_ins_prediction[word_ins_mask]
-                )
-
-                if self.dae_ratio > 0:
-                    # we do not perform denoising for the first iteration
-                    corrputed = (
-                        torch.rand(size=(B,), device=prev_output_tokens.device)
-                        < self.dae_ratio
-                    )
-                    corrputed_tokens = _sequential_poisoning(
-                        tgt_tokens[corrputed],
-                        len(self.tgt_dict),
-                        0.33,
-                        self.bos,
-                        self.eos,
-                        self.pad,
-                    )
-                    prev_output_tokens[corrputed] = corrputed_tokens
-
-        # concat everything
-        word_ins_out = torch.cat(word_ins_outs, 0)
-        word_ins_tgt = torch.cat(word_ins_tgts, 0)
-        word_ins_mask = torch.cat(word_ins_masks, 0)
-
-        return {
-            "word_ins": {
-                "out": word_ins_out,
-                "tgt": word_ins_tgt,
-                "mask": word_ins_mask,
-                "ls": self.args.label_smoothing,
-                "nll_loss": True,
-            },
-            "length": {
-                "out": length_out,
-                "tgt": length_tgt,
-                "factor": self.decoder.length_loss_factor,
-            },
-        }
-
-
-@register_model_architecture(
-    "iterative_nonautoregressive_transformer", "iterative_nonautoregressive_transformer"
-)
-def inat_base_architecture(args):
-    args.encoder_embed_path = getattr(args, "encoder_embed_path", None)
-    args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 512)
-    args.encoder_ffn_embed_dim = getattr(args, "encoder_ffn_embed_dim", 2048)
-    args.encoder_layers = getattr(args, "encoder_layers", 6)
-    args.encoder_attention_heads = getattr(args, "encoder_attention_heads", 8)
-    args.encoder_normalize_before = getattr(args, "encoder_normalize_before", False)
-    args.encoder_learned_pos = getattr(args, "encoder_learned_pos", False)
-    args.decoder_embed_path = getattr(args, "decoder_embed_path", None)
-    args.decoder_embed_dim = getattr(args, "decoder_embed_dim", args.encoder_embed_dim)
-    args.decoder_ffn_embed_dim = getattr(
-        args, "decoder_ffn_embed_dim", args.encoder_ffn_embed_dim
-    )
-    args.decoder_layers = getattr(args, "decoder_layers", 6)
-    args.decoder_attention_heads = getattr(args, "decoder_attention_heads", 8)
-    args.decoder_normalize_before = getattr(args, "decoder_normalize_before", False)
-    args.decoder_learned_pos = getattr(args, "decoder_learned_pos", False)
-    args.attention_dropout = getattr(args, "attention_dropout", 0.0)
-    args.activation_dropout = getattr(args, "activation_dropout", 0.0)
-    args.activation_fn = getattr(args, "activation_fn", "relu")
-    args.dropout = getattr(args, "dropout", 0.1)
-    args.adaptive_softmax_cutoff = getattr(args, "adaptive_softmax_cutoff", None)
-    args.adaptive_softmax_dropout = getattr(args, "adaptive_softmax_dropout", 0)
-    args.share_decoder_input_output_embed = getattr(
-        args, "share_decoder_input_output_embed", False
-    )
-    args.share_all_embeddings = getattr(args, "share_all_embeddings", False)
-    args.no_token_positional_embeddings = getattr(
-        args, "no_token_positional_embeddings", False
-    )
-    args.adaptive_input = getattr(args, "adaptive_input", False)
-    args.apply_bert_init = getattr(args, "apply_bert_init", False)
-
-    args.decoder_output_dim = getattr(
-        args, "decoder_output_dim", args.decoder_embed_dim
-    )
-    args.decoder_input_dim = getattr(args, "decoder_input_dim", args.decoder_embed_dim)
-
-    # --- special arguments ---
-    args.sg_length_pred = getattr(args, "sg_length_pred", False)
-    args.pred_length_offset = getattr(args, "pred_length_offset", False)
-    args.length_loss_factor = getattr(args, "length_loss_factor", 0.1)
-    args.ngram_predictor = getattr(args, "ngram_predictor", 1)
-    args.src_embedding_copy = getattr(args, "src_embedding_copy", False)
-
-    args.train_step = getattr(args, "train_step", 4)
-    args.dae_ratio = getattr(args, "dae_ratio", 0.5)
-    args.stochastic_approx = getattr(args, "stochastic_approx", False)
-
-
-@register_model_architecture(
-    "iterative_nonautoregressive_transformer",
-    "iterative_nonautoregressive_transformer_wmt_en_de",
-)
-def iter_nat_wmt_en_de(args):
-    inat_base_architecture(args)

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/nat/levenshtein_transformer.py

@@ -1,510 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-from fairseq.iterative_refinement_generator import DecoderOut
-from fairseq.models import register_model, register_model_architecture
-from fairseq.models.nat import FairseqNATDecoder, FairseqNATModel, ensemble_decoder
-from fairseq.models.transformer import Embedding
-from fairseq.modules import TransformerDecoderLayer
-from fairseq.modules.transformer_sentence_encoder import init_bert_params
-
-from .levenshtein_utils import (
-    _apply_del_words,
-    _apply_ins_masks,
-    _apply_ins_words,
-    _fill,
-    _get_del_targets,
-    _get_ins_targets,
-    _skip,
-    _skip_encoder_out,
-)
-
-
-@register_model("levenshtein_transformer")
-class LevenshteinTransformerModel(FairseqNATModel):
-    @property
-    def allow_length_beam(self):
-        return False
-
-    @staticmethod
-    def add_args(parser):
-        FairseqNATModel.add_args(parser)
-        parser.add_argument(
-            "--early-exit",
-            default="6,6,6",
-            type=str,
-            help="number of decoder layers before word_del, mask_ins, word_ins",
-        )
-        parser.add_argument(
-            "--no-share-discriminator",
-            action="store_true",
-            help="separate parameters for discriminator",
-        )
-        parser.add_argument(
-            "--no-share-maskpredictor",
-            action="store_true",
-            help="separate parameters for mask-predictor",
-        )
-        parser.add_argument(
-            "--share-discriminator-maskpredictor",
-            action="store_true",
-            help="share the parameters for both mask-predictor and discriminator",
-        )
-        parser.add_argument(
-            "--sampling-for-deletion",
-            action="store_true",
-            help="instead of argmax, use sampling to predict the tokens",
-        )
-
-    @classmethod
-    def build_decoder(cls, args, tgt_dict, embed_tokens):
-        decoder = LevenshteinTransformerDecoder(args, tgt_dict, embed_tokens)
-        if getattr(args, "apply_bert_init", False):
-            decoder.apply(init_bert_params)
-        return decoder
-
-    def forward(
-        self, src_tokens, src_lengths, prev_output_tokens, tgt_tokens, **kwargs
-    ):
-
-        assert tgt_tokens is not None, "forward function only supports training."
-
-        # encoding
-        encoder_out = self.encoder(src_tokens, src_lengths=src_lengths, **kwargs)
-
-        # generate training labels for insertion
-        masked_tgt_masks, masked_tgt_tokens, mask_ins_targets = _get_ins_targets(
-            prev_output_tokens, tgt_tokens, self.pad, self.unk
-        )
-        mask_ins_targets = mask_ins_targets.clamp(min=0, max=255)  # for safe prediction
-        mask_ins_masks = prev_output_tokens[:, 1:].ne(self.pad)
-
-        mask_ins_out, _ = self.decoder.forward_mask_ins(
-            normalize=False,
-            prev_output_tokens=prev_output_tokens,
-            encoder_out=encoder_out,
-        )
-        word_ins_out, _ = self.decoder.forward_word_ins(
-            normalize=False,
-            prev_output_tokens=masked_tgt_tokens,
-            encoder_out=encoder_out,
-        )
-
-        # make online prediction
-        if self.decoder.sampling_for_deletion:
-            word_predictions = torch.multinomial(
-                F.softmax(word_ins_out, -1).view(-1, word_ins_out.size(-1)), 1
-            ).view(word_ins_out.size(0), -1)
-        else:
-            word_predictions = F.log_softmax(word_ins_out, dim=-1).max(2)[1]
-
-        word_predictions.masked_scatter_(
-            ~masked_tgt_masks, tgt_tokens[~masked_tgt_masks]
-        )
-
-        # generate training labels for deletion
-        word_del_targets = _get_del_targets(word_predictions, tgt_tokens, self.pad)
-        word_del_out, _ = self.decoder.forward_word_del(
-            normalize=False,
-            prev_output_tokens=word_predictions,
-            encoder_out=encoder_out,
-        )
-        word_del_masks = word_predictions.ne(self.pad)
-
-        return {
-            "mask_ins": {
-                "out": mask_ins_out,
-                "tgt": mask_ins_targets,
-                "mask": mask_ins_masks,
-                "ls": 0.01,
-            },
-            "word_ins": {
-                "out": word_ins_out,
-                "tgt": tgt_tokens,
-                "mask": masked_tgt_masks,
-                "ls": self.args.label_smoothing,
-                "nll_loss": True,
-            },
-            "word_del": {
-                "out": word_del_out,
-                "tgt": word_del_targets,
-                "mask": word_del_masks,
-            },
-        }
-
-    def forward_decoder(
-        self, decoder_out, encoder_out, eos_penalty=0.0, max_ratio=None, **kwargs
-    ):
-
-        output_tokens = decoder_out.output_tokens
-        output_scores = decoder_out.output_scores
-        attn = decoder_out.attn
-        history = decoder_out.history
-
-        bsz = output_tokens.size(0)
-        if max_ratio is None:
-            max_lens = torch.zeros_like(output_tokens).fill_(255)
-        else:
-            if not encoder_out["encoder_padding_mask"]:
-                max_src_len = encoder_out["encoder_out"].size(0)
-                src_lens = encoder_out["encoder_out"].new(bsz).fill_(max_src_len)
-            else:
-                src_lens = (~encoder_out["encoder_padding_mask"][0]).sum(1)
-            max_lens = (src_lens * max_ratio).clamp(min=10).long()
-
-        # delete words
-        # do not delete tokens if it is <s> </s>
-        can_del_word = output_tokens.ne(self.pad).sum(1) > 2
-        if can_del_word.sum() != 0:  # we cannot delete, skip
-            word_del_score, word_del_attn = self.decoder.forward_word_del(
-                normalize=True,
-                prev_output_tokens=_skip(output_tokens, can_del_word),
-                encoder_out=_skip_encoder_out(self.encoder, encoder_out, can_del_word),
-            )
-            word_del_pred = word_del_score.max(-1)[1].bool()
-
-            _tokens, _scores, _attn = _apply_del_words(
-                output_tokens[can_del_word],
-                output_scores[can_del_word],
-                word_del_attn,
-                word_del_pred,
-                self.pad,
-                self.bos,
-                self.eos,
-            )
-            output_tokens = _fill(output_tokens, can_del_word, _tokens, self.pad)
-            output_scores = _fill(output_scores, can_del_word, _scores, 0)
-            attn = _fill(attn, can_del_word, _attn, 0.0)
-
-            if history is not None:
-                history.append(output_tokens.clone())
-
-        # insert placeholders
-        can_ins_mask = output_tokens.ne(self.pad).sum(1) < max_lens
-        if can_ins_mask.sum() != 0:
-            mask_ins_score, _ = self.decoder.forward_mask_ins(
-                normalize=True,
-                prev_output_tokens=_skip(output_tokens, can_ins_mask),
-                encoder_out=_skip_encoder_out(self.encoder, encoder_out, can_ins_mask),
-            )
-            if eos_penalty > 0.0:
-                mask_ins_score[:, :, 0] = mask_ins_score[:, :, 0] - eos_penalty
-            mask_ins_pred = mask_ins_score.max(-1)[1]
-            mask_ins_pred = torch.min(
-                mask_ins_pred, max_lens[can_ins_mask, None].expand_as(mask_ins_pred)
-            )
-
-            _tokens, _scores = _apply_ins_masks(
-                output_tokens[can_ins_mask],
-                output_scores[can_ins_mask],
-                mask_ins_pred,
-                self.pad,
-                self.unk,
-                self.eos,
-            )
-            output_tokens = _fill(output_tokens, can_ins_mask, _tokens, self.pad)
-            output_scores = _fill(output_scores, can_ins_mask, _scores, 0)
-
-            if history is not None:
-                history.append(output_tokens.clone())
-
-        # insert words
-        can_ins_word = output_tokens.eq(self.unk).sum(1) > 0
-        if can_ins_word.sum() != 0:
-            word_ins_score, word_ins_attn = self.decoder.forward_word_ins(
-                normalize=True,
-                prev_output_tokens=_skip(output_tokens, can_ins_word),
-                encoder_out=_skip_encoder_out(self.encoder, encoder_out, can_ins_word),
-            )
-            word_ins_score, word_ins_pred = word_ins_score.max(-1)
-            _tokens, _scores = _apply_ins_words(
-                output_tokens[can_ins_word],
-                output_scores[can_ins_word],
-                word_ins_pred,
-                word_ins_score,
-                self.unk,
-            )
-
-            output_tokens = _fill(output_tokens, can_ins_word, _tokens, self.pad)
-            output_scores = _fill(output_scores, can_ins_word, _scores, 0)
-            attn = _fill(attn, can_ins_word, word_ins_attn, 0.0)
-
-            if history is not None:
-                history.append(output_tokens.clone())
-
-        # delete some unnecessary paddings
-        cut_off = output_tokens.ne(self.pad).sum(1).max()
-        output_tokens = output_tokens[:, :cut_off]
-        output_scores = output_scores[:, :cut_off]
-        attn = None if attn is None else attn[:, :cut_off, :]
-
-        return decoder_out._replace(
-            output_tokens=output_tokens,
-            output_scores=output_scores,
-            attn=attn,
-            history=history,
-        )
-
-    def initialize_output_tokens(self, encoder_out, src_tokens):
-        initial_output_tokens = src_tokens.new_zeros(src_tokens.size(0), 2)
-        initial_output_tokens[:, 0] = self.bos
-        initial_output_tokens[:, 1] = self.eos
-
-        initial_output_scores = initial_output_tokens.new_zeros(
-            *initial_output_tokens.size()
-        ).type_as(encoder_out["encoder_out"][0])
-
-        return DecoderOut(
-            output_tokens=initial_output_tokens,
-            output_scores=initial_output_scores,
-            attn=None,
-            step=0,
-            max_step=0,
-            history=None,
-        )
-
-
-class LevenshteinTransformerDecoder(FairseqNATDecoder):
-    def __init__(self, args, dictionary, embed_tokens, no_encoder_attn=False):
-        super().__init__(
-            args, dictionary, embed_tokens, no_encoder_attn=no_encoder_attn
-        )
-        self.dictionary = dictionary
-        self.bos = dictionary.bos()
-        self.unk = dictionary.unk()
-        self.eos = dictionary.eos()
-        self.sampling_for_deletion = getattr(args, "sampling_for_deletion", False)
-        self.embed_mask_ins = Embedding(256, self.output_embed_dim * 2, None)
-        self.embed_word_del = Embedding(2, self.output_embed_dim, None)
-
-        # del_word, ins_mask, ins_word
-        self.early_exit = [int(i) for i in args.early_exit.split(",")]
-        assert len(self.early_exit) == 3
-
-        # copy layers for mask-predict/deletion
-        self.layers_msk = None
-        if getattr(args, "no_share_maskpredictor", False):
-            self.layers_msk = nn.ModuleList(
-                [
-                    TransformerDecoderLayer(args, no_encoder_attn)
-                    for _ in range(self.early_exit[1])
-                ]
-            )
-        self.layers_del = None
-        if getattr(args, "no_share_discriminator", False):
-            self.layers_del = nn.ModuleList(
-                [
-                    TransformerDecoderLayer(args, no_encoder_attn)
-                    for _ in range(self.early_exit[0])
-                ]
-            )
-
-        if getattr(args, "share_discriminator_maskpredictor", False):
-            assert getattr(
-                args, "no_share_discriminator", False
-            ), "must set saperate discriminator"
-            self.layers_msk = self.layers_del
-
-    def extract_features(
-        self,
-        prev_output_tokens,
-        encoder_out=None,
-        early_exit=None,
-        layers=None,
-        **unused
-    ):
-        """
-        Similar to *forward* but only return features.
-        Inputs:
-            prev_output_tokens: Tensor(B, T)
-            encoder_out: a dictionary of hidden states and masks
-
-        Returns:
-            tuple:
-                - the decoder's features of shape `(batch, tgt_len, embed_dim)`
-                - a dictionary with any model-specific outputs
-            the LevenshteinTransformer decoder has full-attention to all generated tokens
-        """
-        # embed positions
-        positions = (
-            self.embed_positions(prev_output_tokens)
-            if self.embed_positions is not None
-            else None
-        )
-
-        # embed tokens and positions
-        x = self.embed_scale * self.embed_tokens(prev_output_tokens)
-        if self.project_in_dim is not None:
-            x = self.project_in_dim(x)
-
-        if positions is not None:
-            x += positions
-        x = self.dropout_module(x)
-
-        # B x T x C -> T x B x C
-        x = x.transpose(0, 1)
-        attn = None
-        inner_states = [x]
-
-        # decoder layers
-        decoder_padding_mask = prev_output_tokens.eq(self.padding_idx)
-        layers = self.layers if layers is None else layers
-        early_exit = len(layers) if early_exit is None else early_exit
-        for _, layer in enumerate(layers[:early_exit]):
-            x, attn, _ = layer(
-                x,
-                encoder_out["encoder_out"][0]
-                if (encoder_out is not None and len(encoder_out["encoder_out"]) > 0)
-                else None,
-                encoder_out["encoder_padding_mask"][0]
-                if (
-                    encoder_out is not None
-                    and len(encoder_out["encoder_padding_mask"]) > 0
-                )
-                else None,
-                self_attn_mask=None,
-                self_attn_padding_mask=decoder_padding_mask,
-            )
-            inner_states.append(x)
-
-        if self.layer_norm:
-            x = self.layer_norm(x)
-
-        # T x B x C -> B x T x C
-        x = x.transpose(0, 1)
-
-        if self.project_out_dim is not None:
-            x = self.project_out_dim(x)
-
-        return x, {"attn": attn, "inner_states": inner_states}
-
-    @ensemble_decoder
-    def forward_mask_ins(self, normalize, encoder_out, prev_output_tokens, **unused):
-        features, extra = self.extract_features(
-            prev_output_tokens,
-            encoder_out=encoder_out,
-            early_exit=self.early_exit[1],
-            layers=self.layers_msk,
-            **unused
-        )
-        features_cat = torch.cat([features[:, :-1, :], features[:, 1:, :]], 2)
-        decoder_out = F.linear(features_cat, self.embed_mask_ins.weight)
-        if normalize:
-            return F.log_softmax(decoder_out, -1), extra["attn"]
-        return decoder_out, extra["attn"]
-
-    @ensemble_decoder
-    def forward_word_ins(self, normalize, encoder_out, prev_output_tokens, **unused):
-        features, extra = self.extract_features(
-            prev_output_tokens,
-            encoder_out=encoder_out,
-            early_exit=self.early_exit[2],
-            layers=self.layers,
-            **unused
-        )
-        decoder_out = self.output_layer(features)
-        if normalize:
-            return F.log_softmax(decoder_out, -1), extra["attn"]
-        return decoder_out, extra["attn"]
-
-    @ensemble_decoder
-    def forward_word_del(self, normalize, encoder_out, prev_output_tokens, **unused):
-        features, extra = self.extract_features(
-            prev_output_tokens,
-            encoder_out=encoder_out,
-            early_exit=self.early_exit[0],
-            layers=self.layers_del,
-            **unused
-        )
-        decoder_out = F.linear(features, self.embed_word_del.weight)
-        if normalize:
-            return F.log_softmax(decoder_out, -1), extra["attn"]
-        return decoder_out, extra["attn"]
-
-
-@register_model_architecture("levenshtein_transformer", "levenshtein_transformer")
-def levenshtein_base_architecture(args):
-    args.encoder_embed_path = getattr(args, "encoder_embed_path", None)
-    args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 512)
-    args.encoder_ffn_embed_dim = getattr(args, "encoder_ffn_embed_dim", 2048)
-    args.encoder_layers = getattr(args, "encoder_layers", 6)
-    args.encoder_attention_heads = getattr(args, "encoder_attention_heads", 8)
-    args.encoder_normalize_before = getattr(args, "encoder_normalize_before", False)
-    args.encoder_learned_pos = getattr(args, "encoder_learned_pos", False)
-    args.decoder_embed_path = getattr(args, "decoder_embed_path", None)
-    args.decoder_embed_dim = getattr(args, "decoder_embed_dim", args.encoder_embed_dim)
-    args.decoder_ffn_embed_dim = getattr(
-        args, "decoder_ffn_embed_dim", args.encoder_ffn_embed_dim
-    )
-    args.decoder_layers = getattr(args, "decoder_layers", 6)
-    args.decoder_attention_heads = getattr(args, "decoder_attention_heads", 8)
-    args.decoder_normalize_before = getattr(args, "decoder_normalize_before", False)
-    args.decoder_learned_pos = getattr(args, "decoder_learned_pos", False)
-    args.attention_dropout = getattr(args, "attention_dropout", 0.0)
-    args.activation_dropout = getattr(args, "activation_dropout", 0.0)
-    args.activation_fn = getattr(args, "activation_fn", "relu")
-    args.dropout = getattr(args, "dropout", 0.1)
-    args.adaptive_softmax_cutoff = getattr(args, "adaptive_softmax_cutoff", None)
-    args.adaptive_softmax_dropout = getattr(args, "adaptive_softmax_dropout", 0)
-    args.share_decoder_input_output_embed = getattr(
-        args, "share_decoder_input_output_embed", False
-    )
-    args.share_all_embeddings = getattr(args, "share_all_embeddings", False)
-    args.no_token_positional_embeddings = getattr(
-        args, "no_token_positional_embeddings", False
-    )
-    args.adaptive_input = getattr(args, "adaptive_input", False)
-    args.apply_bert_init = getattr(args, "apply_bert_init", False)
-
-    args.decoder_output_dim = getattr(
-        args, "decoder_output_dim", args.decoder_embed_dim
-    )
-    args.sampling_for_deletion = getattr(args, "sampling_for_deletion", False)
-    args.decoder_input_dim = getattr(args, "decoder_input_dim", args.decoder_embed_dim)
-    args.early_exit = getattr(args, "early_exit", "6,6,6")
-    args.no_share_discriminator = getattr(args, "no_share_discriminator", False)
-    args.no_share_maskpredictor = getattr(args, "no_share_maskpredictor", False)
-    args.share_discriminator_maskpredictor = getattr(
-        args, "share_discriminator_maskpredictor", False
-    )
-    args.no_share_last_layer = getattr(args, "no_share_last_layer", False)
-
-
-@register_model_architecture(
-    "levenshtein_transformer", "levenshtein_transformer_wmt_en_de"
-)
-def levenshtein_transformer_wmt_en_de(args):
-    levenshtein_base_architecture(args)
-
-
-# similar parameters used in the "Attention Is All You Need" paper (Vaswani et al., 2017)
-@register_model_architecture(
-    "levenshtein_transformer", "levenshtein_transformer_vaswani_wmt_en_de_big"
-)
-def levenshtein_transformer_vaswani_wmt_en_de_big(args):
-    args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 1024)
-    args.encoder_ffn_embed_dim = getattr(args, "encoder_ffn_embed_dim", 4096)
-    args.encoder_attention_heads = getattr(args, "encoder_attention_heads", 16)
-    args.encoder_normalize_before = getattr(args, "encoder_normalize_before", False)
-    args.decoder_embed_dim = getattr(args, "decoder_embed_dim", 1024)
-    args.decoder_ffn_embed_dim = getattr(args, "decoder_ffn_embed_dim", 4096)
-    args.decoder_attention_heads = getattr(args, "decoder_attention_heads", 16)
-    args.dropout = getattr(args, "dropout", 0.3)
-    levenshtein_base_architecture(args)
-
-
-# default parameters used in tensor2tensor implementation
-@register_model_architecture(
-    "levenshtein_transformer", "levenshtein_transformer_wmt_en_de_big"
-)
-def levenshtein_transformer_wmt_en_de_big_t2t(args):
-    args.encoder_normalize_before = getattr(args, "encoder_normalize_before", True)
-    args.decoder_normalize_before = getattr(args, "decoder_normalize_before", True)
-    args.attention_dropout = getattr(args, "attention_dropout", 0.1)
-    args.activation_dropout = getattr(args, "activation_dropout", 0.1)
-    levenshtein_transformer_vaswani_wmt_en_de_big(args)

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/nat/levenshtein_utils.py

@@ -1,292 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-import torch
-from fairseq.utils import new_arange
-
-# -------------- Helper Functions --------------------------------------------------- #
-
-
-def load_libnat():
-    try:
-        from fairseq import libnat_cuda
-
-        return libnat_cuda, True
-
-    except ImportError as e:
-        print(str(e) + "... fall back to CPU version")
-
-        try:
-            from fairseq import libnat
-
-            return libnat, False
-
-        except ImportError as e:
-            import sys
-
-            sys.stderr.write(
-                "ERROR: missing libnat_cuda. run `python setup.py build_ext --inplace`\n"
-            )
-            raise e
-
-
-def _get_ins_targets(in_tokens, out_tokens, padding_idx, unk_idx):
-    libnat, use_cuda = load_libnat()
-
-    def _get_ins_targets_cuda(in_tokens, out_tokens, padding_idx, unk_idx):
-        in_masks = in_tokens.ne(padding_idx)
-        out_masks = out_tokens.ne(padding_idx)
-        mask_ins_targets, masked_tgt_masks = libnat.generate_insertion_labels(
-            out_tokens.int(),
-            libnat.levenshtein_distance(
-                in_tokens.int(),
-                out_tokens.int(),
-                in_masks.sum(1).int(),
-                out_masks.sum(1).int(),
-            ),
-        )
-        masked_tgt_masks = masked_tgt_masks.bool() & out_masks
-        mask_ins_targets = mask_ins_targets.type_as(in_tokens)[
-            :, 1 : in_masks.size(1)
-        ].masked_fill_(~in_masks[:, 1:], 0)
-        masked_tgt_tokens = out_tokens.masked_fill(masked_tgt_masks, unk_idx)
-        return masked_tgt_masks, masked_tgt_tokens, mask_ins_targets
-
-    def _get_ins_targets_cpu(in_tokens, out_tokens, padding_idx, unk_idx):
-        in_seq_len, out_seq_len = in_tokens.size(1), out_tokens.size(1)
-
-        in_tokens_list = [
-            [t for t in s if t != padding_idx] for i, s in enumerate(in_tokens.tolist())
-        ]
-        out_tokens_list = [
-            [t for t in s if t != padding_idx]
-            for i, s in enumerate(out_tokens.tolist())
-        ]
-
-        full_labels = libnat.suggested_ed2_path(
-            in_tokens_list, out_tokens_list, padding_idx
-        )
-        mask_inputs = [
-            [len(c) if c[0] != padding_idx else 0 for c in a[:-1]] for a in full_labels
-        ]
-
-        # generate labels
-        masked_tgt_masks = []
-        for mask_input in mask_inputs:
-            mask_label = []
-            for beam_size in mask_input[1:-1]:  # HACK 1:-1
-                mask_label += [0] + [1 for _ in range(beam_size)]
-            masked_tgt_masks.append(
-                mask_label + [0 for _ in range(out_seq_len - len(mask_label))]
-            )
-        mask_ins_targets = [
-            mask_input[1:-1]
-            + [0 for _ in range(in_seq_len - 1 - len(mask_input[1:-1]))]
-            for mask_input in mask_inputs
-        ]
-
-        # transform to tensor
-        masked_tgt_masks = torch.tensor(
-            masked_tgt_masks, device=out_tokens.device
-        ).bool()
-        mask_ins_targets = torch.tensor(mask_ins_targets, device=in_tokens.device)
-        masked_tgt_tokens = out_tokens.masked_fill(masked_tgt_masks, unk_idx)
-        return masked_tgt_masks, masked_tgt_tokens, mask_ins_targets
-
-    if use_cuda:
-        return _get_ins_targets_cuda(in_tokens, out_tokens, padding_idx, unk_idx)
-    return _get_ins_targets_cpu(in_tokens, out_tokens, padding_idx, unk_idx)
-
-
-def _get_del_targets(in_tokens, out_tokens, padding_idx):
-    libnat, use_cuda = load_libnat()
-
-    def _get_del_targets_cuda(in_tokens, out_tokens, padding_idx):
-        in_masks = in_tokens.ne(padding_idx)
-        out_masks = out_tokens.ne(padding_idx)
-
-        word_del_targets = libnat.generate_deletion_labels(
-            in_tokens.int(),
-            libnat.levenshtein_distance(
-                in_tokens.int(),
-                out_tokens.int(),
-                in_masks.sum(1).int(),
-                out_masks.sum(1).int(),
-            ),
-        )
-        word_del_targets = word_del_targets.type_as(in_tokens).masked_fill_(
-            ~in_masks, 0
-        )
-        return word_del_targets
-
-    def _get_del_targets_cpu(in_tokens, out_tokens, padding_idx):
-        out_seq_len = out_tokens.size(1)
-        with torch.cuda.device_of(in_tokens):
-            in_tokens_list = [
-                [t for t in s if t != padding_idx]
-                for i, s in enumerate(in_tokens.tolist())
-            ]
-            out_tokens_list = [
-                [t for t in s if t != padding_idx]
-                for i, s in enumerate(out_tokens.tolist())
-            ]
-
-        full_labels = libnat.suggested_ed2_path(
-            in_tokens_list, out_tokens_list, padding_idx
-        )
-        word_del_targets = [b[-1] for b in full_labels]
-        word_del_targets = [
-            labels + [0 for _ in range(out_seq_len - len(labels))]
-            for labels in word_del_targets
-        ]
-
-        # transform to tensor
-        word_del_targets = torch.tensor(word_del_targets, device=out_tokens.device)
-        return word_del_targets
-
-    if use_cuda:
-        return _get_del_targets_cuda(in_tokens, out_tokens, padding_idx)
-    return _get_del_targets_cpu(in_tokens, out_tokens, padding_idx)
-
-
-def _apply_ins_masks(
-    in_tokens, in_scores, mask_ins_pred, padding_idx, unk_idx, eos_idx
-):
-
-    in_masks = in_tokens.ne(padding_idx)
-    in_lengths = in_masks.sum(1)
-
-    # HACK: hacky way to shift all the paddings to eos first.
-    in_tokens.masked_fill_(~in_masks, eos_idx)
-    mask_ins_pred.masked_fill_(~in_masks[:, 1:], 0)
-
-    out_lengths = in_lengths + mask_ins_pred.sum(1)
-    out_max_len = out_lengths.max()
-    out_masks = new_arange(out_lengths, out_max_len)[None, :] < out_lengths[:, None]
-
-    reordering = (mask_ins_pred + in_masks[:, 1:].long()).cumsum(1)
-    out_tokens = (
-        in_tokens.new_zeros(in_tokens.size(0), out_max_len)
-        .fill_(padding_idx)
-        .masked_fill_(out_masks, unk_idx)
-    )
-    out_tokens[:, 0] = in_tokens[:, 0]
-    out_tokens.scatter_(1, reordering, in_tokens[:, 1:])
-
-    out_scores = None
-    if in_scores is not None:
-        in_scores.masked_fill_(~in_masks, 0)
-        out_scores = in_scores.new_zeros(*out_tokens.size())
-        out_scores[:, 0] = in_scores[:, 0]
-        out_scores.scatter_(1, reordering, in_scores[:, 1:])
-
-    return out_tokens, out_scores
-
-
-def _apply_ins_words(in_tokens, in_scores, word_ins_pred, word_ins_scores, unk_idx):
-    word_ins_masks = in_tokens.eq(unk_idx)
-    out_tokens = in_tokens.masked_scatter(word_ins_masks, word_ins_pred[word_ins_masks])
-
-    if in_scores is not None:
-        out_scores = in_scores.masked_scatter(
-            word_ins_masks, word_ins_scores[word_ins_masks]
-        )
-    else:
-        out_scores = None
-
-    return out_tokens, out_scores
-
-
-def _apply_del_words(
-    in_tokens, in_scores, in_attn, word_del_pred, padding_idx, bos_idx, eos_idx
-):
-    # apply deletion to a tensor
-    in_masks = in_tokens.ne(padding_idx)
-    bos_eos_masks = in_tokens.eq(bos_idx) | in_tokens.eq(eos_idx)
-
-    max_len = in_tokens.size(1)
-    word_del_pred.masked_fill_(~in_masks, 1)
-    word_del_pred.masked_fill_(bos_eos_masks, 0)
-
-    reordering = new_arange(in_tokens).masked_fill_(word_del_pred, max_len).sort(1)[1]
-
-    out_tokens = in_tokens.masked_fill(word_del_pred, padding_idx).gather(1, reordering)
-
-    out_scores = None
-    if in_scores is not None:
-        out_scores = in_scores.masked_fill(word_del_pred, 0).gather(1, reordering)
-
-    out_attn = None
-    if in_attn is not None:
-        _mask = word_del_pred[:, :, None].expand_as(in_attn)
-        _reordering = reordering[:, :, None].expand_as(in_attn)
-        out_attn = in_attn.masked_fill(_mask, 0.0).gather(1, _reordering)
-
-    return out_tokens, out_scores, out_attn
-
-
-def _skip(x, mask):
-    """
-    Getting sliced (dim=0) tensor by mask. Supporting tensor and list/dict of tensors.
-    """
-    if isinstance(x, int):
-        return x
-
-    if x is None:
-        return None
-
-    if isinstance(x, torch.Tensor):
-        if x.size(0) == mask.size(0):
-            return x[mask]
-        elif x.size(1) == mask.size(0):
-            return x[:, mask]
-
-    if isinstance(x, list):
-        return [_skip(x_i, mask) for x_i in x]
-
-    if isinstance(x, dict):
-        return {k: _skip(v, mask) for k, v in x.items()}
-
-    raise NotImplementedError
-
-
-def _skip_encoder_out(encoder, encoder_out, mask):
-    if not mask.any():
-        return encoder_out
-    else:
-        return encoder.reorder_encoder_out(
-            encoder_out, mask.nonzero(as_tuple=False).squeeze()
-        )
-
-
-def _fill(x, mask, y, padding_idx):
-    """
-    Filling tensor x with y at masked positions (dim=0).
-    """
-    if x is None:
-        return y
-    assert x.dim() == y.dim() and mask.size(0) == x.size(0)
-    assert x.dim() == 2 or (x.dim() == 3 and x.size(2) == y.size(2))
-    n_selected = mask.sum()
-    assert n_selected == y.size(0)
-
-    if n_selected == x.size(0):
-        return y
-
-    if x.size(1) < y.size(1):
-        dims = [x.size(0), y.size(1) - x.size(1)]
-        if x.dim() == 3:
-            dims.append(x.size(2))
-        x = torch.cat([x, x.new_zeros(*dims).fill_(padding_idx)], 1)
-        x[mask] = y
-    elif x.size(1) > y.size(1):
-        x[mask] = padding_idx
-        if x.dim() == 2:
-            x[mask, : y.size(1)] = y
-        else:
-            x[mask, : y.size(1), :] = y
-    else:
-        x[mask] = y
-    return x

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/nat/nat_crf_transformer.py

@@ -1,121 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-
-from fairseq.models import register_model, register_model_architecture
-from fairseq.models.nat import NATransformerModel, base_architecture
-from fairseq.modules import DynamicCRF
-
-
-@register_model("nacrf_transformer")
-class NACRFTransformerModel(NATransformerModel):
-    def __init__(self, args, encoder, decoder):
-        super().__init__(args, encoder, decoder)
-        self.crf_layer = DynamicCRF(
-            num_embedding=len(self.tgt_dict),
-            low_rank=args.crf_lowrank_approx,
-            beam_size=args.crf_beam_approx,
-        )
-
-    @property
-    def allow_ensemble(self):
-        return False
-
-    @staticmethod
-    def add_args(parser):
-        NATransformerModel.add_args(parser)
-        parser.add_argument(
-            "--crf-lowrank-approx",
-            type=int,
-            help="the dimension of low-rank approximation of transition",
-        )
-        parser.add_argument(
-            "--crf-beam-approx",
-            type=int,
-            help="the beam size for apporixmating the normalizing factor",
-        )
-        parser.add_argument(
-            "--word-ins-loss-factor",
-            type=float,
-            help="weights on NAT loss used to co-training with CRF loss.",
-        )
-
-    def forward(
-        self, src_tokens, src_lengths, prev_output_tokens, tgt_tokens, **kwargs
-    ):
-        # encoding
-        encoder_out = self.encoder(src_tokens, src_lengths=src_lengths, **kwargs)
-
-        # length prediction
-        length_out = self.decoder.forward_length(
-            normalize=False, encoder_out=encoder_out
-        )
-        length_tgt = self.decoder.forward_length_prediction(
-            length_out, encoder_out, tgt_tokens
-        )
-
-        # decoding
-        word_ins_out = self.decoder(
-            normalize=False,
-            prev_output_tokens=prev_output_tokens,
-            encoder_out=encoder_out,
-        )
-        word_ins_tgt, word_ins_mask = tgt_tokens, tgt_tokens.ne(self.pad)
-
-        # compute the log-likelihood of CRF
-        crf_nll = -self.crf_layer(word_ins_out, word_ins_tgt, word_ins_mask)
-        crf_nll = (crf_nll / word_ins_mask.type_as(crf_nll).sum(-1)).mean()
-
-        return {
-            "word_ins": {
-                "out": word_ins_out,
-                "tgt": word_ins_tgt,
-                "mask": word_ins_mask,
-                "ls": self.args.label_smoothing,
-                "nll_loss": True,
-                "factor": self.args.word_ins_loss_factor,
-            },
-            "word_crf": {"loss": crf_nll},
-            "length": {
-                "out": length_out,
-                "tgt": length_tgt,
-                "factor": self.decoder.length_loss_factor,
-            },
-        }
-
-    def forward_decoder(self, decoder_out, encoder_out, decoding_format=None, **kwargs):
-        output_tokens = decoder_out.output_tokens
-        output_scores = decoder_out.output_scores
-        history = decoder_out.history
-
-        # execute the decoder and get emission scores
-        output_masks = output_tokens.ne(self.pad)
-        word_ins_out = self.decoder(
-            normalize=False, prev_output_tokens=output_tokens, encoder_out=encoder_out
-        )
-
-        # run viterbi decoding through CRF
-        _scores, _tokens = self.crf_layer.forward_decoder(word_ins_out, output_masks)
-        output_tokens.masked_scatter_(output_masks, _tokens[output_masks])
-        output_scores.masked_scatter_(output_masks, _scores[output_masks])
-        if history is not None:
-            history.append(output_tokens.clone())
-
-        return decoder_out._replace(
-            output_tokens=output_tokens,
-            output_scores=output_scores,
-            attn=None,
-            history=history,
-        )
-
-
-@register_model_architecture("nacrf_transformer", "nacrf_transformer")
-def nacrf_base_architecture(args):
-    args.crf_lowrank_approx = getattr(args, "crf_lowrank_approx", 32)
-    args.crf_beam_approx = getattr(args, "crf_beam_approx", 64)
-    args.word_ins_loss_factor = getattr(args, "word_ins_loss_factor", 0.5)
-    args.encoder_normalize_before = getattr(args, "encoder_normalize_before", True)
-    args.decoder_normalize_before = getattr(args, "decoder_normalize_before", True)
-    base_architecture(args)

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/nat/nonautoregressive_ensembles.py

@@ -1,254 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-import math
-
-import torch
-import torch.nn.functional as F
-from fairseq.models.nat import (
-    _apply_del_words,
-    _apply_ins_masks,
-    _apply_ins_words,
-    _fill,
-    _skip,
-    _skip_encoder_out,
-)
-
-
-class _EnsembleModelEncoder(object):
-    def __init__(self, models):
-        self.models = models
-
-    def reorder_encoder_out(self, encoder_outs, new_order):
-        encoder_outs = [
-            model.encoder.reorder_encoder_out(encoder_out, new_order)
-            for model, encoder_out in zip(self.models, encoder_outs)
-        ]
-        return encoder_outs
-
-
-class BasicEnsembleModel(torch.nn.Module):
-    """A wrapper around an ensemble of models."""
-
-    def __init__(self, models):
-        super().__init__()
-        self.models = torch.nn.ModuleList(models)
-        self.bos = self.models[0].decoder.dictionary.bos()
-        self.eos = self.models[0].decoder.dictionary.eos()
-        self.pad = self.models[0].decoder.dictionary.pad()
-        self.unk = self.models[0].decoder.dictionary.unk()
-        self.encoder = _EnsembleModelEncoder(self.models)
-
-    def has_encoder(self):
-        return hasattr(self.models[0], "encoder")
-
-    def max_decoder_positions(self):
-        return min(m.max_decoder_positions() for m in self.models)
-
-    @torch.no_grad()
-    def forward_encoder(self, encoder_input):
-        if not self.has_encoder():
-            return None
-        return [model.forward_encoder(encoder_input) for model in self.models]
-
-    @torch.no_grad()
-    def forward_decoder(self, *inputs):
-        raise NotImplementedError
-
-    def initialize_output_tokens(self, *inputs):
-        raise NotImplementedError
-
-
-class EnsembleLevT(BasicEnsembleModel):
-    """A wrapper around an ensemble of models."""
-
-    def __init__(self, models):
-        super().__init__(models)
-
-    @torch.no_grad()
-    def forward_decoder(
-        self, decoder_out, encoder_outs, eos_penalty=0.0, max_ratio=None, **kwargs
-    ):
-        # LevT ensembling
-        # A pipeline of three steps: deletion, placeholder, and word insertion.
-        # We need to average scores in each step in a pipeline way because of dependence.
-        # deletion
-        output_tokens = decoder_out.output_tokens
-        output_scores = decoder_out.output_scores
-        attn = decoder_out.attn
-
-        bsz = output_tokens.size(0)
-        if max_ratio is None:
-            max_lens = output_tokens.new().fill_(255)
-        else:
-            if not encoder_outs[0]["encoder_padding_mask"]:
-                src_lens = (
-                    encoder_outs[0]["encoder_out"][0]
-                    .new(bsz)
-                    .fill_(encoder_outs[0]["encoder_out"][0].size(1))
-                )
-            else:
-                src_lens = (~encoder_outs[0]["encoder_padding_mask"][0]).sum(1)
-            max_lens = (src_lens * max_ratio).clamp(min=10).long()
-
-        # delete words
-        # do not delete tokens if it is <s> </s>
-        can_del_word = output_tokens.ne(self.pad).sum(1) > 2
-        if can_del_word.sum() != 0:  # we cannot delete, skip
-            output_tokens, output_scores, attn = self.forward_word_del(
-                encoder_outs,
-                output_tokens,
-                output_scores,
-                attn,
-                can_del_word,
-            )
-
-        # insert placeholders
-        can_ins_mask = output_tokens.ne(self.pad).sum(1) < max_lens
-        if can_ins_mask.sum() != 0:
-            output_tokens, output_scores = self.forward_mask_ins(
-                encoder_outs,
-                output_tokens,
-                output_scores,
-                can_ins_mask,
-                eos_penalty,
-                max_lens,
-            )
-
-        # insert words
-        can_ins_word = output_tokens.eq(self.unk).sum(1) > 0
-        if can_ins_word.sum() != 0:
-            output_tokens, output_scores, attn = self.forward_word_ins(
-                encoder_outs,
-                output_tokens,
-                output_scores,
-                attn,
-                can_ins_word,
-            )
-
-        # delete some unnecessary paddings
-        cut_off = output_tokens.ne(self.pad).sum(1).max()
-        output_tokens = output_tokens[:, :cut_off]
-        output_scores = output_scores[:, :cut_off]
-        attn = None if attn is None else attn[:, :cut_off, :]
-        return decoder_out._replace(
-            output_tokens=output_tokens,
-            output_scores=output_scores,
-            attn=attn,
-            history=None,
-        )
-
-    def forward_word_del(
-        self, encoder_outs, output_tokens, output_scores, attn, can_del_word
-    ):
-        word_del_score_avg = []
-        word_del_attn_avg = []
-        for model, encoder_out in zip(self.models, encoder_outs):
-            word_del_out, word_del_attn = model.decoder.forward_word_del(
-                _skip(output_tokens, can_del_word),
-                _skip_encoder_out(model.encoder, encoder_out, can_del_word),
-            )
-            word_del_score = F.log_softmax(word_del_out, 2)
-            word_del_score_avg.append(word_del_score)
-            word_del_attn_avg.append(word_del_attn)
-        word_del_score_avg = torch.logsumexp(
-            torch.stack(word_del_score_avg, dim=0), dim=0
-        ) - math.log(len(self.models))
-        word_del_pred = word_del_score_avg.max(-1)[1].bool()
-        if word_del_attn_avg[0] is not None:
-            word_del_attn_avg = torch.stack(word_del_attn_avg, dim=0) / len(self.models)
-        else:
-            word_del_attn_avg = None
-
-        _tokens, _scores, _attn = _apply_del_words(
-            output_tokens[can_del_word],
-            output_scores[can_del_word],
-            word_del_attn_avg,
-            word_del_pred,
-            self.pad,
-            self.bos,
-            self.eos,
-        )
-        output_tokens = _fill(output_tokens, can_del_word, _tokens, self.pad)
-        output_scores = _fill(output_scores, can_del_word, _scores, 0)
-        attn = _fill(attn, can_del_word, _attn, 0.0)
-        return output_tokens, output_scores, attn
-
-    def forward_mask_ins(
-        self,
-        encoder_outs,
-        output_tokens,
-        output_scores,
-        can_ins_mask,
-        eos_penalty,
-        max_lens,
-    ):
-        mask_ins_score_avg = []
-        for model, encoder_out in zip(self.models, encoder_outs):
-            mask_ins_out, _ = model.decoder.forward_mask_ins(
-                _skip(output_tokens, can_ins_mask),
-                _skip_encoder_out(model.encoder, encoder_out, can_ins_mask),
-            )
-            mask_ins_score = F.log_softmax(mask_ins_out, 2)
-            if eos_penalty > 0.0:
-                mask_ins_score[:, :, 0] -= eos_penalty
-            mask_ins_score_avg.append(mask_ins_score)
-        mask_ins_score_avg = torch.logsumexp(
-            torch.stack(mask_ins_score_avg, dim=0), dim=0
-        ) - math.log(len(self.models))
-        mask_ins_pred = mask_ins_score_avg.max(-1)[1]
-        mask_ins_pred = torch.min(
-            mask_ins_pred, max_lens[can_ins_mask, None].expand_as(mask_ins_pred)
-        )
-        _tokens, _scores = _apply_ins_masks(
-            output_tokens[can_ins_mask],
-            output_scores[can_ins_mask],
-            mask_ins_pred,
-            self.pad,
-            self.unk,
-            self.eos,
-        )
-        output_tokens = _fill(output_tokens, can_ins_mask, _tokens, self.pad)
-        output_scores = _fill(output_scores, can_ins_mask, _scores, 0)
-        return output_tokens, output_scores
-
-    def forward_word_ins(
-        self, encoder_outs, output_tokens, output_scores, attn, can_ins_word
-    ):
-        word_ins_score_avg = []
-        word_ins_attn_avg = []
-        for model, encoder_out in zip(self.models, encoder_outs):
-            word_ins_out, word_ins_attn = model.decoder.forward_word_ins(
-                _skip(output_tokens, can_ins_word),
-                _skip_encoder_out(model.encoder, encoder_out, can_ins_word),
-            )
-            word_ins_score = F.log_softmax(word_ins_out, 2)
-            word_ins_score_avg.append(word_ins_score)
-            word_ins_attn_avg.append(word_ins_attn)
-        word_ins_score_avg = torch.logsumexp(
-            torch.stack(word_ins_score_avg, dim=0), dim=0
-        ) - math.log(len(self.models))
-        if word_ins_attn_avg[0] is not None:
-            word_ins_attn_avg = torch.stack(word_ins_attn_avg, dim=0) / len(self.models)
-        else:
-            word_ins_attn_avg = None
-        word_ins_score_max, word_ins_pred = word_ins_score_avg.max(-1)
-
-        _tokens, _scores = _apply_ins_words(
-            output_tokens[can_ins_word],
-            output_scores[can_ins_word],
-            word_ins_pred,
-            word_ins_score_max,
-            self.unk,
-        )
-
-        output_tokens = _fill(output_tokens, can_ins_word, _tokens, self.pad)
-        output_scores = _fill(output_scores, can_ins_word, _scores, 0)
-        attn = _fill(attn, can_ins_word, word_ins_attn, 0.0)
-        return output_tokens, output_scores, attn
-
-    def initialize_output_tokens(self, encoder_outs, src_tokens):
-        # LevT doesn't do length prediction.
-        return self.models[0].initialize_output_tokens(encoder_outs[0], src_tokens)

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/nat/nonautoregressive_transformer.py

@@ -1,456 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-import torch
-import torch.nn.functional as F
-from fairseq import utils
-from fairseq.iterative_refinement_generator import DecoderOut
-from fairseq.models import register_model, register_model_architecture
-from fairseq.models.nat import FairseqNATDecoder, FairseqNATModel, ensemble_decoder
-from fairseq.models.transformer import Embedding
-from fairseq.modules.transformer_sentence_encoder import init_bert_params
-
-
-def _mean_pooling(enc_feats, src_masks):
-    # enc_feats: T x B x C
-    # src_masks: B x T or None
-    if src_masks is None:
-        enc_feats = enc_feats.mean(0)
-    else:
-        src_masks = (~src_masks).transpose(0, 1).type_as(enc_feats)
-        enc_feats = (
-            (enc_feats / src_masks.sum(0)[None, :, None]) * src_masks[:, :, None]
-        ).sum(0)
-    return enc_feats
-
-
-def _argmax(x, dim):
-    return (x == x.max(dim, keepdim=True)[0]).type_as(x)
-
-
-def _uniform_assignment(src_lens, trg_lens):
-    max_trg_len = trg_lens.max()
-    steps = (src_lens.float() - 1) / (trg_lens.float() - 1)  # step-size
-    # max_trg_len
-    index_t = utils.new_arange(trg_lens, max_trg_len).float()
-    index_t = steps[:, None] * index_t[None, :]  # batch_size X max_trg_len
-    index_t = torch.round(index_t).long().detach()
-    return index_t
-
-
-@register_model("nonautoregressive_transformer")
-class NATransformerModel(FairseqNATModel):
-    @property
-    def allow_length_beam(self):
-        return True
-
-    @staticmethod
-    def add_args(parser):
-        FairseqNATModel.add_args(parser)
-
-        # length prediction
-        parser.add_argument(
-            "--src-embedding-copy",
-            action="store_true",
-            help="copy encoder word embeddings as the initial input of the decoder",
-        )
-        parser.add_argument(
-            "--pred-length-offset",
-            action="store_true",
-            help="predicting the length difference between the target and source sentences",
-        )
-        parser.add_argument(
-            "--sg-length-pred",
-            action="store_true",
-            help="stop the gradients back-propagated from the length predictor",
-        )
-        parser.add_argument(
-            "--length-loss-factor",
-            type=float,
-            help="weights on the length prediction loss",
-        )
-
-    @classmethod
-    def build_decoder(cls, args, tgt_dict, embed_tokens):
-        decoder = NATransformerDecoder(args, tgt_dict, embed_tokens)
-        if getattr(args, "apply_bert_init", False):
-            decoder.apply(init_bert_params)
-        return decoder
-
-    def forward(
-        self, src_tokens, src_lengths, prev_output_tokens, tgt_tokens, **kwargs
-    ):
-        # encoding
-        encoder_out = self.encoder(src_tokens, src_lengths=src_lengths, **kwargs)
-
-        # length prediction
-        length_out = self.decoder.forward_length(
-            normalize=False, encoder_out=encoder_out
-        )
-        length_tgt = self.decoder.forward_length_prediction(
-            length_out, encoder_out, tgt_tokens
-        )
-
-        # decoding
-        word_ins_out = self.decoder(
-            normalize=False,
-            prev_output_tokens=prev_output_tokens,
-            encoder_out=encoder_out,
-        )
-
-        return {
-            "word_ins": {
-                "out": word_ins_out,
-                "tgt": tgt_tokens,
-                "mask": tgt_tokens.ne(self.pad),
-                "ls": self.args.label_smoothing,
-                "nll_loss": True,
-            },
-            "length": {
-                "out": length_out,
-                "tgt": length_tgt,
-                "factor": self.decoder.length_loss_factor,
-            },
-        }
-
-    def forward_decoder(self, decoder_out, encoder_out, decoding_format=None, **kwargs):
-        step = decoder_out.step
-        output_tokens = decoder_out.output_tokens
-        output_scores = decoder_out.output_scores
-        history = decoder_out.history
-
-        # execute the decoder
-        output_masks = output_tokens.ne(self.pad)
-        _scores, _tokens = self.decoder(
-            normalize=True,
-            prev_output_tokens=output_tokens,
-            encoder_out=encoder_out,
-            step=step,
-        ).max(-1)
-
-        output_tokens.masked_scatter_(output_masks, _tokens[output_masks])
-        output_scores.masked_scatter_(output_masks, _scores[output_masks])
-        if history is not None:
-            history.append(output_tokens.clone())
-
-        return decoder_out._replace(
-            output_tokens=output_tokens,
-            output_scores=output_scores,
-            attn=None,
-            history=history,
-        )
-
-    def initialize_output_tokens(self, encoder_out, src_tokens):
-        # length prediction
-        length_tgt = self.decoder.forward_length_prediction(
-            self.decoder.forward_length(normalize=True, encoder_out=encoder_out),
-            encoder_out=encoder_out,
-        )
-
-        max_length = length_tgt.clamp_(min=2).max()
-        idx_length = utils.new_arange(src_tokens, max_length)
-
-        initial_output_tokens = src_tokens.new_zeros(
-            src_tokens.size(0), max_length
-        ).fill_(self.pad)
-        initial_output_tokens.masked_fill_(
-            idx_length[None, :] < length_tgt[:, None], self.unk
-        )
-        initial_output_tokens[:, 0] = self.bos
-        initial_output_tokens.scatter_(1, length_tgt[:, None] - 1, self.eos)
-
-        initial_output_scores = initial_output_tokens.new_zeros(
-            *initial_output_tokens.size()
-        ).type_as(encoder_out["encoder_out"][0])
-
-        return DecoderOut(
-            output_tokens=initial_output_tokens,
-            output_scores=initial_output_scores,
-            attn=None,
-            step=0,
-            max_step=0,
-            history=None,
-        )
-
-    def regenerate_length_beam(self, decoder_out, beam_size):
-        output_tokens = decoder_out.output_tokens
-        length_tgt = output_tokens.ne(self.pad).sum(1)
-        length_tgt = (
-            length_tgt[:, None]
-            + utils.new_arange(length_tgt, 1, beam_size)
-            - beam_size // 2
-        )
-        length_tgt = length_tgt.view(-1).clamp_(min=2)
-        max_length = length_tgt.max()
-        idx_length = utils.new_arange(length_tgt, max_length)
-
-        initial_output_tokens = output_tokens.new_zeros(
-            length_tgt.size(0), max_length
-        ).fill_(self.pad)
-        initial_output_tokens.masked_fill_(
-            idx_length[None, :] < length_tgt[:, None], self.unk
-        )
-        initial_output_tokens[:, 0] = self.bos
-        initial_output_tokens.scatter_(1, length_tgt[:, None] - 1, self.eos)
-
-        initial_output_scores = initial_output_tokens.new_zeros(
-            *initial_output_tokens.size()
-        ).type_as(decoder_out.output_scores)
-
-        return decoder_out._replace(
-            output_tokens=initial_output_tokens, output_scores=initial_output_scores
-        )
-
-
-class NATransformerDecoder(FairseqNATDecoder):
-    def __init__(self, args, dictionary, embed_tokens, no_encoder_attn=False):
-        super().__init__(
-            args, dictionary, embed_tokens, no_encoder_attn=no_encoder_attn
-        )
-        self.dictionary = dictionary
-        self.bos = dictionary.bos()
-        self.unk = dictionary.unk()
-        self.eos = dictionary.eos()
-
-        self.encoder_embed_dim = args.encoder_embed_dim
-        self.sg_length_pred = getattr(args, "sg_length_pred", False)
-        self.pred_length_offset = getattr(args, "pred_length_offset", False)
-        self.length_loss_factor = getattr(args, "length_loss_factor", 0.1)
-        self.src_embedding_copy = getattr(args, "src_embedding_copy", False)
-        self.embed_length = Embedding(256, self.encoder_embed_dim, None)
-
-    @ensemble_decoder
-    def forward(self, normalize, encoder_out, prev_output_tokens, step=0, **unused):
-        features, _ = self.extract_features(
-            prev_output_tokens,
-            encoder_out=encoder_out,
-            embedding_copy=(step == 0) & self.src_embedding_copy,
-        )
-        decoder_out = self.output_layer(features)
-        return F.log_softmax(decoder_out, -1) if normalize else decoder_out
-
-    @ensemble_decoder
-    def forward_length(self, normalize, encoder_out):
-        enc_feats = encoder_out["encoder_out"][0]  # T x B x C
-        if len(encoder_out["encoder_padding_mask"]) > 0:
-            src_masks = encoder_out["encoder_padding_mask"][0]  # B x T
-        else:
-            src_masks = None
-        enc_feats = _mean_pooling(enc_feats, src_masks)
-        if self.sg_length_pred:
-            enc_feats = enc_feats.detach()
-        length_out = F.linear(enc_feats, self.embed_length.weight)
-        return F.log_softmax(length_out, -1) if normalize else length_out
-
-    def extract_features(
-        self,
-        prev_output_tokens,
-        encoder_out=None,
-        early_exit=None,
-        embedding_copy=False,
-        **unused
-    ):
-        """
-        Similar to *forward* but only return features.
-
-        Inputs:
-            prev_output_tokens: Tensor(B, T)
-            encoder_out: a dictionary of hidden states and masks
-
-        Returns:
-            tuple:
-                - the decoder's features of shape `(batch, tgt_len, embed_dim)`
-                - a dictionary with any model-specific outputs
-            the LevenshteinTransformer decoder has full-attention to all generated tokens
-        """
-        # embedding
-        if embedding_copy:
-            src_embd = encoder_out["encoder_embedding"][0]
-            if len(encoder_out["encoder_padding_mask"]) > 0:
-                src_mask = encoder_out["encoder_padding_mask"][0]
-            else:
-                src_mask = None
-            src_mask = (
-                ~src_mask
-                if src_mask is not None
-                else prev_output_tokens.new_ones(*src_embd.size()[:2]).bool()
-            )
-
-            x, decoder_padding_mask = self.forward_embedding(
-                prev_output_tokens,
-                self.forward_copying_source(
-                    src_embd, src_mask, prev_output_tokens.ne(self.padding_idx)
-                ),
-            )
-
-        else:
-
-            x, decoder_padding_mask = self.forward_embedding(prev_output_tokens)
-
-        # B x T x C -> T x B x C
-        x = x.transpose(0, 1)
-        attn = None
-        inner_states = [x]
-
-        # decoder layers
-        for i, layer in enumerate(self.layers):
-
-            # early exit from the decoder.
-            if (early_exit is not None) and (i >= early_exit):
-                break
-
-            x, attn, _ = layer(
-                x,
-                encoder_out["encoder_out"][0]
-                if (encoder_out is not None and len(encoder_out["encoder_out"]) > 0)
-                else None,
-                encoder_out["encoder_padding_mask"][0]
-                if (
-                    encoder_out is not None
-                    and len(encoder_out["encoder_padding_mask"]) > 0
-                )
-                else None,
-                self_attn_mask=None,
-                self_attn_padding_mask=decoder_padding_mask,
-            )
-            inner_states.append(x)
-
-        if self.layer_norm:
-            x = self.layer_norm(x)
-
-        # T x B x C -> B x T x C
-        x = x.transpose(0, 1)
-
-        if self.project_out_dim is not None:
-            x = self.project_out_dim(x)
-
-        return x, {"attn": attn, "inner_states": inner_states}
-
-    def forward_embedding(self, prev_output_tokens, states=None):
-        # embed positions
-        positions = (
-            self.embed_positions(prev_output_tokens)
-            if self.embed_positions is not None
-            else None
-        )
-
-        # embed tokens and positions
-        if states is None:
-            x = self.embed_scale * self.embed_tokens(prev_output_tokens)
-            if self.project_in_dim is not None:
-                x = self.project_in_dim(x)
-        else:
-            x = states
-
-        if positions is not None:
-            x += positions
-        x = self.dropout_module(x)
-        decoder_padding_mask = prev_output_tokens.eq(self.padding_idx)
-        return x, decoder_padding_mask
-
-    def forward_copying_source(self, src_embeds, src_masks, tgt_masks):
-        length_sources = src_masks.sum(1)
-        length_targets = tgt_masks.sum(1)
-        mapped_inputs = _uniform_assignment(length_sources, length_targets).masked_fill(
-            ~tgt_masks, 0
-        )
-        copied_embedding = torch.gather(
-            src_embeds,
-            1,
-            mapped_inputs.unsqueeze(-1).expand(
-                *mapped_inputs.size(), src_embeds.size(-1)
-            ),
-        )
-        return copied_embedding
-
-    def forward_length_prediction(self, length_out, encoder_out, tgt_tokens=None):
-        enc_feats = encoder_out["encoder_out"][0]  # T x B x C
-        if len(encoder_out["encoder_padding_mask"]) > 0:
-            src_masks = encoder_out["encoder_padding_mask"][0]  # B x T
-        else:
-            src_masks = None
-        if self.pred_length_offset:
-            if src_masks is None:
-                src_lengs = enc_feats.new_ones(enc_feats.size(1)).fill_(
-                    enc_feats.size(0)
-                )
-            else:
-                src_lengs = (~src_masks).transpose(0, 1).type_as(enc_feats).sum(0)
-            src_lengs = src_lengs.long()
-
-        if tgt_tokens is not None:
-            # obtain the length target
-            tgt_lengs = tgt_tokens.ne(self.padding_idx).sum(1).long()
-            if self.pred_length_offset:
-                length_tgt = tgt_lengs - src_lengs + 128
-            else:
-                length_tgt = tgt_lengs
-            length_tgt = length_tgt.clamp(min=0, max=255)
-
-        else:
-            # predict the length target (greedy for now)
-            # TODO: implementing length-beam
-            pred_lengs = length_out.max(-1)[1]
-            if self.pred_length_offset:
-                length_tgt = pred_lengs - 128 + src_lengs
-            else:
-                length_tgt = pred_lengs
-
-        return length_tgt
-
-
-@register_model_architecture(
-    "nonautoregressive_transformer", "nonautoregressive_transformer"
-)
-def base_architecture(args):
-    args.encoder_embed_path = getattr(args, "encoder_embed_path", None)
-    args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 512)
-    args.encoder_ffn_embed_dim = getattr(args, "encoder_ffn_embed_dim", 2048)
-    args.encoder_layers = getattr(args, "encoder_layers", 6)
-    args.encoder_attention_heads = getattr(args, "encoder_attention_heads", 8)
-    args.encoder_normalize_before = getattr(args, "encoder_normalize_before", False)
-    args.encoder_learned_pos = getattr(args, "encoder_learned_pos", False)
-    args.decoder_embed_path = getattr(args, "decoder_embed_path", None)
-    args.decoder_embed_dim = getattr(args, "decoder_embed_dim", args.encoder_embed_dim)
-    args.decoder_ffn_embed_dim = getattr(
-        args, "decoder_ffn_embed_dim", args.encoder_ffn_embed_dim
-    )
-    args.decoder_layers = getattr(args, "decoder_layers", 6)
-    args.decoder_attention_heads = getattr(args, "decoder_attention_heads", 8)
-    args.decoder_normalize_before = getattr(args, "decoder_normalize_before", False)
-    args.decoder_learned_pos = getattr(args, "decoder_learned_pos", False)
-    args.attention_dropout = getattr(args, "attention_dropout", 0.0)
-    args.activation_dropout = getattr(args, "activation_dropout", 0.0)
-    args.activation_fn = getattr(args, "activation_fn", "relu")
-    args.dropout = getattr(args, "dropout", 0.1)
-    args.adaptive_softmax_cutoff = getattr(args, "adaptive_softmax_cutoff", None)
-    args.adaptive_softmax_dropout = getattr(args, "adaptive_softmax_dropout", 0)
-    args.share_decoder_input_output_embed = getattr(
-        args, "share_decoder_input_output_embed", False
-    )
-    args.share_all_embeddings = getattr(args, "share_all_embeddings", False)
-    args.no_token_positional_embeddings = getattr(
-        args, "no_token_positional_embeddings", False
-    )
-    args.adaptive_input = getattr(args, "adaptive_input", False)
-    args.apply_bert_init = getattr(args, "apply_bert_init", False)
-
-    args.decoder_output_dim = getattr(
-        args, "decoder_output_dim", args.decoder_embed_dim
-    )
-    args.decoder_input_dim = getattr(args, "decoder_input_dim", args.decoder_embed_dim)
-
-    # --- special arguments ---
-    args.sg_length_pred = getattr(args, "sg_length_pred", False)
-    args.pred_length_offset = getattr(args, "pred_length_offset", False)
-    args.length_loss_factor = getattr(args, "length_loss_factor", 0.1)
-    args.src_embedding_copy = getattr(args, "src_embedding_copy", False)
-
-
-@register_model_architecture(
-    "nonautoregressive_transformer", "nonautoregressive_transformer_wmt_en_de"
-)
-def nonautoregressive_transformer_wmt_en_de(args):
-    base_architecture(args)

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/roberta/__init__.py

@@ -1,11 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-from .enc_dec import *  # noqa
-from .hub_interface import *  # noqa
-from .model import *  # noqa
-from .model_camembert import *  # noqa
-from .model_gottbert import *  # noqa
-from .model_xlmr import *  # noqa

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/roberta/alignment_utils.py

@@ -1,118 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-from collections import Counter
-from typing import List
-
-import torch
-
-
-def align_bpe_to_words(roberta, bpe_tokens: torch.LongTensor, other_tokens: List[str]):
-    """
-    Helper to align GPT-2 BPE to other tokenization formats (e.g., spaCy).
-
-    Args:
-        roberta (RobertaHubInterface): RoBERTa instance
-        bpe_tokens (torch.LongTensor): GPT-2 BPE tokens of shape `(T_bpe)`
-        other_tokens (List[str]): other tokens of shape `(T_words)`
-
-    Returns:
-        List[str]: mapping from *other_tokens* to corresponding *bpe_tokens*.
-    """
-    assert bpe_tokens.dim() == 1
-    assert bpe_tokens[0] == 0
-
-    def clean(text):
-        return text.strip()
-
-    # remove whitespaces to simplify alignment
-    bpe_tokens = [roberta.task.source_dictionary.string([x]) for x in bpe_tokens]
-    bpe_tokens = [
-        clean(roberta.bpe.decode(x) if x not in {"<s>", ""} else x) for x in bpe_tokens
-    ]
-    other_tokens = [clean(str(o)) for o in other_tokens]
-
-    # strip leading <s>
-    bpe_tokens = bpe_tokens[1:]
-    assert "".join(bpe_tokens) == "".join(other_tokens)
-
-    # create alignment from every word to a list of BPE tokens
-    alignment = []
-    bpe_toks = filter(lambda item: item[1] != "", enumerate(bpe_tokens, start=1))
-    j, bpe_tok = next(bpe_toks)
-    for other_tok in other_tokens:
-        bpe_indices = []
-        while True:
-            if other_tok.startswith(bpe_tok):
-                bpe_indices.append(j)
-                other_tok = other_tok[len(bpe_tok) :]
-                try:
-                    j, bpe_tok = next(bpe_toks)
-                except StopIteration:
-                    j, bpe_tok = None, None
-            elif bpe_tok.startswith(other_tok):
-                # other_tok spans multiple BPE tokens
-                bpe_indices.append(j)
-                bpe_tok = bpe_tok[len(other_tok) :]
-                other_tok = ""
-            else:
-                raise Exception('Cannot align "{}" and "{}"'.format(other_tok, bpe_tok))
-            if other_tok == "":
-                break
-        assert len(bpe_indices) > 0
-        alignment.append(bpe_indices)
-    assert len(alignment) == len(other_tokens)
-
-    return alignment
-
-
-def align_features_to_words(roberta, features, alignment):
-    """
-    Align given features to words.
-
-    Args:
-        roberta (RobertaHubInterface): RoBERTa instance
-        features (torch.Tensor): features to align of shape `(T_bpe x C)`
-        alignment: alignment between BPE tokens and words returned by
-            func:`align_bpe_to_words`.
-    """
-    assert features.dim() == 2
-
-    bpe_counts = Counter(j for bpe_indices in alignment for j in bpe_indices)
-    assert bpe_counts[0] == 0  # <s> shouldn't be aligned
-    denom = features.new([bpe_counts.get(j, 1) for j in range(len(features))])
-    weighted_features = features / denom.unsqueeze(-1)
-
-    output = [weighted_features[0]]
-    largest_j = -1
-    for bpe_indices in alignment:
-        output.append(weighted_features[bpe_indices].sum(dim=0))
-        largest_j = max(largest_j, *bpe_indices)
-    for j in range(largest_j + 1, len(features)):
-        output.append(weighted_features[j])
-    output = torch.stack(output)
-    assert torch.all(torch.abs(output.sum(dim=0) - features.sum(dim=0)) < 1e-4)
-    return output
-
-
-def spacy_nlp():
-    if getattr(spacy_nlp, "_nlp", None) is None:
-        try:
-            from spacy.lang.en import English
-
-            spacy_nlp._nlp = English()
-        except ImportError:
-            raise ImportError("Please install spacy with: pip install spacy")
-    return spacy_nlp._nlp
-
-
-def spacy_tokenizer():
-    if getattr(spacy_tokenizer, "_tokenizer", None) is None:
-        try:
-            nlp = spacy_nlp()
-            spacy_tokenizer._tokenizer = nlp.Defaults.create_tokenizer(nlp)
-        except ImportError:
-            raise ImportError("Please install spacy with: pip install spacy")
-    return spacy_tokenizer._tokenizer

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/roberta/enc_dec.py

@@ -1,192 +0,0 @@
-import argparse
-import logging
-
-import fairseq.checkpoint_utils
-import torch.nn as nn
-from fairseq.models import (
-    FairseqEncoderDecoderModel,
-    register_model,
-    register_model_architecture,
-)
-from fairseq.models.roberta import model as roberta
-from fairseq.models.transformer import TransformerDecoder
-
-logger = logging.getLogger(__name__)
-
-
-@register_model("roberta_enc_dec")
-class RobertaEncDecModel(FairseqEncoderDecoderModel):
-    @staticmethod
-    def add_args(parser):
-        parser.add_argument(
-            "--pretrained-mlm-checkpoint",
-            default=None,
-            type=str,
-            metavar="PRETRAINED",
-            help="path to pretrained mlm checkpoint",
-        )
-        parser.add_argument(
-            "--pretrained-decoder", action="store_true", help="reload decoder"
-        )
-        parser.add_argument(
-            "--hack-layernorm-embedding",
-            action="store_true",
-            help="hack to reload old models trained with encoder-normalize-before=False (no equivalent to encoder-normalize-before=False and layernorm_embedding=False",
-        )
-        parser.add_argument(
-            "--share-decoder-input-output-embed",
-            action="store_true",
-            help="share decoder input and output embeddings",
-        )
-        parser.add_argument(
-            "--share-all-embeddings",
-            action="store_true",
-            help="share encoder, decoder and output embeddings"
-            " (requires shared dictionary and embed dim)",
-        )
-
-    @classmethod
-    def build_model(cls, args, task):
-        """Build a new model instance."""
-
-        # make sure all arguments are present
-        base_enc_dec_architecture(args)
-        if args.pretrained_mlm_checkpoint:
-            arg_overrides = None
-            if args.hack_layernorm_embedding:
-                arg_overrides = {"layernorm_embedding": False}
-            loaded = fairseq.checkpoint_utils.load_model_ensemble_and_task(
-                [args.pretrained_mlm_checkpoint], arg_overrides=arg_overrides
-            )
-            ([roberta_enc], _cfg, _task) = loaded
-        else:
-            # Do we need to edit untie_weights here ?
-            share_in_out = (
-                args.share_decoder_input_output_embed or args.share_all_embeddings
-            )
-            args.untie_weights_roberta = not share_in_out
-            if args.hack_layernorm_embedding:
-                args.layernorm_embedding = False
-                args.encoder_normalize_before = False
-            roberta_enc = roberta.RobertaModel.build_model(args, task)
-
-        return cls.from_roberta(roberta_enc, args, task.source_dictionary)
-
-    @staticmethod
-    def from_roberta(roberta_enc: roberta.RobertaModel, args, dictionary):
-        encoder = roberta_enc.encoder.sentence_encoder
-        vocab_size, embed_dim = encoder.embed_tokens.weight.shape
-
-        if args.share_all_embeddings:
-            lm_head = roberta_enc.encoder.lm_head
-            assert encoder.embed_tokens.weight is lm_head.weight, (
-                "Can't use --share-all-embeddings with a model "
-                "that was pretraiend with --untie-weights-roberta_enc"
-            )
-        else:
-            lm_head = roberta.RobertaLMHead(
-                embed_dim, vocab_size, roberta_enc.args.activation_fn
-            )
-
-        dec_embs = nn.Embedding(vocab_size, embed_dim, dictionary.pad())
-        if args.share_all_embeddings or args.share_decoder_input_output_embed:
-            # Note: I wasn't able to use Embedding _weight parameter to achive this sharing.
-            dec_embs.weight = lm_head.weight
-
-        decoder = TransformerDecoder(
-            RobertaEncDecModel.read_args_from_roberta(roberta_enc.args),
-            dictionary,
-            dec_embs,
-            no_encoder_attn=False,
-            output_projection=lm_head,
-        )
-        if getattr(args, "pretrained_decoder", False):
-            decoder_dict = encoder.state_dict()
-
-            # TODO: hide setting "encoder_attn" layers behind a flag.
-            for k, w in list(decoder_dict.items()):
-                if ".self_attn" in k:
-                    k_enc_attn = k.replace(".self_attn", ".encoder_attn")
-                    decoder_dict[k_enc_attn] = w.detach().clone()
-
-            for k, w in lm_head.state_dict().items():
-                decoder_dict["output_projection." + k] = w
-
-            missing_keys, unexpected_keys = decoder.load_state_dict(
-                decoder_dict, strict=False
-            )
-            # missing_keys = [m for m in missing_keys if ".encoder_attn" not in m]
-            assert not missing_keys and not unexpected_keys, (
-                "Failed to load state dict. "
-                f"Missing keys: {missing_keys}. "
-                f"Unexpected keys: {unexpected_keys}."
-            )
-
-        if args.share_all_embeddings:
-            assert decoder.output_projection.weight is decoder.embed_tokens.weight
-            assert encoder.embed_tokens.weight is decoder.embed_tokens.weight
-        elif args.share_decoder_input_output_embed:
-            assert decoder.output_projection.weight is decoder.embed_tokens.weight
-            assert encoder.embed_tokens.weight is not decoder.embed_tokens.weight
-        else:
-            assert decoder.output_projection.weight is not decoder.embed_tokens.weight
-            assert encoder.embed_tokens.weight is not decoder.embed_tokens.weight
-
-        return RobertaEncDecModel(encoder, decoder)
-
-    @staticmethod
-    def read_args_from_roberta(roberta_args: argparse.Namespace):
-        # TODO: this would become easier if encoder/decoder where using a similar
-        # TransformerConfig object
-        args = argparse.Namespace(**vars(roberta_args))
-        attr_map = [
-            ("encoder_attention_heads", "decoder_attention_heads"),
-            ("encoder_embed_dim", "decoder_embed_dim"),
-            ("encoder_embed_dim", "decoder_output_dim"),
-            ("encoder_normalize_before", "decoder_normalize_before"),
-            ("encoder_layers_to_keep", "decoder_layers_to_keep"),
-            ("encoder_ffn_embed_dim", "decoder_ffn_embed_dim"),
-            ("encoder_layerdrop", "decoder_layerdrop"),
-            ("encoder_layers", "decoder_layers"),
-            ("encoder_learned_pos", "decoder_learned_pos"),
-            # should this be set from here ?
-            ("max_positions", "max_target_positions"),
-        ]
-        for k1, k2 in attr_map:
-            setattr(args, k2, getattr(roberta_args, k1))
-
-        args.adaptive_softmax_cutoff = getattr(args, "adaptive_softmax_cutoff", None)
-        args.adaptive_softmax_dropout = getattr(args, "adaptive_softmax_dropout", 0)
-        args.share_decoder_input_output_embed = not roberta_args.untie_weights_roberta
-        return args
-
-    def upgrade_state_dict_named(self, state_dict, name):
-        prefix = name + "." if name != "" else ""
-        super().upgrade_state_dict_named(state_dict, name)
-        old_keys = list(state_dict.keys())
-
-        # rename decoder -> encoder before upgrading children modules
-        for k in old_keys:
-            if k.startswith(prefix + "encoder.lm_head"):
-                state_dict.pop(k)
-                continue
-            new_k = k
-            new_k = new_k.replace(".sentence_encoder.", ".")
-            new_k = new_k.replace("decoder.lm_head.", "decoder.output_projection.")
-            if k == new_k:
-                continue
-            # print(k, "->", new_k)
-            state_dict[new_k] = state_dict.pop(k)
-
-
-@register_model_architecture("roberta_enc_dec", "roberta_enc_dec")
-def base_enc_dec_architecture(args):
-    args.hack_layernorm_embedding = getattr(args, "hack_layernorm_embedding", False)
-    args.pretrained_mlm_checkpoint = getattr(args, "pretrained_mlm_checkpoint", None)
-    args.pretrained_decoder = getattr(args, "pretrained_decoder", None)
-    args.share_all_embeddings = getattr(args, "share_all_embeddings", False)
-    args.share_decoder_input_output_embed = getattr(
-        args, "share_decoder_input_output_embed", False
-    )
-
-    roberta.base_architecture(args)

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/roberta/hub_interface.py

@@ -1,235 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-import numpy as np
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-from fairseq import utils
-from fairseq.data import encoders
-
-
-class RobertaHubInterface(nn.Module):
-    """A simple PyTorch Hub interface to RoBERTa.
-
-    Usage: https://github.com/pytorch/fairseq/tree/main/examples/roberta
-    """
-
-    def __init__(self, cfg, task, model):
-        super().__init__()
-        self.cfg = cfg
-        self.task = task
-        self.model = model
-
-        self.bpe = encoders.build_bpe(cfg.bpe)
-
-        # this is useful for determining the device
-        self.register_buffer("_float_tensor", torch.tensor([0], dtype=torch.float))
-
-    @property
-    def device(self):
-        return self._float_tensor.device
-
-    def encode(
-        self, sentence: str, *addl_sentences, no_separator=False
-    ) -> torch.LongTensor:
-        """
-        BPE-encode a sentence (or multiple sentences).
-
-        Every sequence begins with a beginning-of-sentence (`<s>`) symbol.
-        Every sentence ends with an end-of-sentence (`</s>`) and we use an
-        extra end-of-sentence (`</s>`) as a separator.
-
-        Example (single sentence): `<s> a b c </s>`
-        Example (sentence pair): `<s> d e f </s> </s> 1 2 3 </s>`
-
-        The BPE encoding follows GPT-2. One subtle detail is that the GPT-2 BPE
-        requires leading spaces. For example::
-
-            >>> roberta.encode('Hello world').tolist()
-            [0, 31414, 232, 2]
-            >>> roberta.encode(' world').tolist()
-            [0, 232, 2]
-            >>> roberta.encode('world').tolist()
-            [0, 8331, 2]
-        """
-        bpe_sentence = "<s> " + self.bpe.encode(sentence) + " </s>"
-        for s in addl_sentences:
-            bpe_sentence += " </s>" if not no_separator else ""
-            bpe_sentence += " " + self.bpe.encode(s) + " </s>"
-        tokens = self.task.source_dictionary.encode_line(
-            bpe_sentence, append_eos=False, add_if_not_exist=False
-        )
-        return tokens.long()
-
-    def decode(self, tokens: torch.LongTensor):
-        assert tokens.dim() == 1
-        tokens = tokens.numpy()
-        if tokens[0] == self.task.source_dictionary.bos():
-            tokens = tokens[1:]  # remove <s>
-        eos_mask = tokens == self.task.source_dictionary.eos()
-        doc_mask = eos_mask[1:] & eos_mask[:-1]
-        sentences = np.split(tokens, doc_mask.nonzero()[0] + 1)
-        sentences = [
-            self.bpe.decode(self.task.source_dictionary.string(s)) for s in sentences
-        ]
-        if len(sentences) == 1:
-            return sentences[0]
-        return sentences
-
-    def extract_features(
-        self, tokens: torch.LongTensor, return_all_hiddens: bool = False
-    ) -> torch.Tensor:
-        if tokens.dim() == 1:
-            tokens = tokens.unsqueeze(0)
-        if tokens.size(-1) > self.model.max_positions():
-            raise ValueError(
-                "tokens exceeds maximum length: {} > {}".format(
-                    tokens.size(-1), self.model.max_positions()
-                )
-            )
-        features, extra = self.model(
-            tokens.to(device=self.device),
-            features_only=True,
-            return_all_hiddens=return_all_hiddens,
-        )
-        if return_all_hiddens:
-            # convert from T x B x C -> B x T x C
-            inner_states = extra["inner_states"]
-            return [inner_state.transpose(0, 1) for inner_state in inner_states]
-        else:
-            return features  # just the last layer's features
-
-    def register_classification_head(
-        self, name: str, num_classes: int = None, embedding_size: int = None, **kwargs
-    ):
-        self.model.register_classification_head(
-            name, num_classes=num_classes, embedding_size=embedding_size, **kwargs
-        )
-
-    def predict(self, head: str, tokens: torch.LongTensor, return_logits: bool = False):
-        features = self.extract_features(tokens.to(device=self.device))
-        logits = self.model.classification_heads[head](features)
-        if return_logits:
-            return logits
-        return F.log_softmax(logits, dim=-1)
-
-    def extract_features_aligned_to_words(
-        self, sentence: str, return_all_hiddens: bool = False
-    ) -> torch.Tensor:
-        """Extract RoBERTa features, aligned to spaCy's word-level tokenizer."""
-        from fairseq.models.roberta import alignment_utils
-        from spacy.tokens import Doc
-
-        nlp = alignment_utils.spacy_nlp()
-        tokenizer = alignment_utils.spacy_tokenizer()
-
-        # tokenize both with GPT-2 BPE and spaCy
-        bpe_toks = self.encode(sentence)
-        spacy_toks = tokenizer(sentence)
-        spacy_toks_ws = [t.text_with_ws for t in tokenizer(sentence)]
-        alignment = alignment_utils.align_bpe_to_words(self, bpe_toks, spacy_toks_ws)
-
-        # extract features and align them
-        features = self.extract_features(
-            bpe_toks, return_all_hiddens=return_all_hiddens
-        )
-        features = features.squeeze(0)
-        aligned_feats = alignment_utils.align_features_to_words(
-            self, features, alignment
-        )
-
-        # wrap in spaCy Doc
-        doc = Doc(
-            nlp.vocab,
-            words=["<s>"] + [x.text for x in spacy_toks] + ["</s>"],
-            spaces=[True]
-            + [x.endswith(" ") for x in spacy_toks_ws[:-1]]
-            + [True, False],
-        )
-        assert len(doc) == aligned_feats.size(0)
-        doc.user_token_hooks["vector"] = lambda token: aligned_feats[token.i]
-        return doc
-
-    def fill_mask(self, masked_input: str, topk: int = 5):
-        masked_token = "<mask>"
-        assert (
-            masked_token in masked_input and masked_input.count(masked_token) == 1
-        ), "Please add one {0} token for the input, eg: 'He is a {0} guy'".format(
-            masked_token
-        )
-
-        text_spans = masked_input.split(masked_token)
-        text_spans_bpe = (
-            (" {0} ".format(masked_token))
-            .join([self.bpe.encode(text_span.rstrip()) for text_span in text_spans])
-            .strip()
-        )
-        tokens = self.task.source_dictionary.encode_line(
-            "<s> " + text_spans_bpe + " </s>",
-            append_eos=False,
-            add_if_not_exist=False,
-        )
-
-        masked_index = (tokens == self.task.mask_idx).nonzero(as_tuple=False)
-        if tokens.dim() == 1:
-            tokens = tokens.unsqueeze(0)
-
-        with utils.model_eval(self.model):
-            features, extra = self.model(
-                tokens.long().to(device=self.device),
-                features_only=False,
-                return_all_hiddens=False,
-            )
-        logits = features[0, masked_index, :].squeeze()
-        prob = logits.softmax(dim=0)
-        values, index = prob.topk(k=topk, dim=0)
-        topk_predicted_token_bpe = self.task.source_dictionary.string(index)
-
-        topk_filled_outputs = []
-        for index, predicted_token_bpe in enumerate(
-            topk_predicted_token_bpe.split(" ")
-        ):
-            predicted_token = self.bpe.decode(predicted_token_bpe)
-            # Quick hack to fix https://github.com/pytorch/fairseq/issues/1306
-            if predicted_token_bpe.startswith("\u2581"):
-                predicted_token = " " + predicted_token
-            if " {0}".format(masked_token) in masked_input:
-                topk_filled_outputs.append(
-                    (
-                        masked_input.replace(
-                            " {0}".format(masked_token), predicted_token
-                        ),
-                        values[index].item(),
-                        predicted_token,
-                    )
-                )
-            else:
-                topk_filled_outputs.append(
-                    (
-                        masked_input.replace(masked_token, predicted_token),
-                        values[index].item(),
-                        predicted_token,
-                    )
-                )
-        return topk_filled_outputs
-
-    def disambiguate_pronoun(self, sentence: str) -> bool:
-        """
-        Usage::
-
-            >>> disambiguate_pronoun('The _trophy_ would not fit in the brown suitcase because [it] was too big.')
-            True
-
-            >>> disambiguate_pronoun('The trophy would not fit in the brown suitcase because [it] was too big.')
-            'The trophy'
-        """
-        assert hasattr(
-            self.task, "disambiguate_pronoun"
-        ), "roberta.disambiguate_pronoun() requires a model trained with the WSC task."
-        with utils.model_eval(self.model):
-            return self.task.disambiguate_pronoun(
-                self.model, sentence, use_cuda=self.device.type == "cuda"
-            )

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/roberta/model.py

@@ -1,699 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-"""
-RoBERTa: A Robustly Optimized BERT Pretraining Approach.
-"""
-
-import logging
-
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-from fairseq import utils
-from fairseq.models import (
-    FairseqEncoder,
-    FairseqEncoderModel,
-    register_model,
-    register_model_architecture,
-)
-from fairseq.models.transformer import DEFAULT_MIN_PARAMS_TO_WRAP, TransformerEncoder
-from fairseq.modules import LayerNorm
-from fairseq.modules.quant_noise import quant_noise as apply_quant_noise_
-from fairseq.modules.transformer_sentence_encoder import init_bert_params
-from fairseq.utils import safe_getattr, safe_hasattr
-
-from .hub_interface import RobertaHubInterface
-
-logger = logging.getLogger(__name__)
-
-
-@register_model("roberta")
-class RobertaModel(FairseqEncoderModel):
-    @classmethod
-    def hub_models(cls):
-        return {
-            "roberta.base": "http://dl.fbaipublicfiles.com/fairseq/models/roberta.base.tar.gz",
-            "roberta.large": "http://dl.fbaipublicfiles.com/fairseq/models/roberta.large.tar.gz",
-            "roberta.large.mnli": "http://dl.fbaipublicfiles.com/fairseq/models/roberta.large.mnli.tar.gz",
-            "roberta.large.wsc": "http://dl.fbaipublicfiles.com/fairseq/models/roberta.large.wsc.tar.gz",
-        }
-
-    def __init__(self, args, encoder):
-        super().__init__(encoder)
-        self.args = args
-
-        # We follow BERT's random weight initialization
-        self.apply(init_bert_params)
-
-        self.classification_heads = nn.ModuleDict()
-
-    @staticmethod
-    def add_args(parser):
-        """Add model-specific arguments to the parser."""
-        parser.add_argument(
-            "--encoder-layers", type=int, metavar="L", help="num encoder layers"
-        )
-        parser.add_argument(
-            "--encoder-embed-dim",
-            type=int,
-            metavar="H",
-            help="encoder embedding dimension",
-        )
-        parser.add_argument(
-            "--encoder-ffn-embed-dim",
-            type=int,
-            metavar="F",
-            help="encoder embedding dimension for FFN",
-        )
-        parser.add_argument(
-            "--encoder-attention-heads",
-            type=int,
-            metavar="A",
-            help="num encoder attention heads",
-        )
-        parser.add_argument(
-            "--activation-fn",
-            choices=utils.get_available_activation_fns(),
-            help="activation function to use",
-        )
-        parser.add_argument(
-            "--pooler-activation-fn",
-            choices=utils.get_available_activation_fns(),
-            help="activation function to use for pooler layer",
-        )
-        parser.add_argument(
-            "--encoder-normalize-before",
-            action="store_true",
-            help="apply layernorm before each encoder block",
-        )
-        parser.add_argument(
-            "--layernorm-embedding",
-            action="store_true",
-            help="add layernorm to embedding",
-        )
-        parser.add_argument(
-            "--dropout", type=float, metavar="D", help="dropout probability"
-        )
-        parser.add_argument(
-            "--attention-dropout",
-            type=float,
-            metavar="D",
-            help="dropout probability for attention weights",
-        )
-        parser.add_argument(
-            "--activation-dropout",
-            type=float,
-            metavar="D",
-            help="dropout probability after activation in FFN",
-        )
-        parser.add_argument(
-            "--pooler-dropout",
-            type=float,
-            metavar="D",
-            help="dropout probability in the masked_lm pooler layers",
-        )
-        parser.add_argument(
-            "--max-positions", type=int, help="number of positional embeddings to learn"
-        )
-        parser.add_argument(
-            "--load-checkpoint-heads",
-            action="store_true",
-            help="(re-)register and load heads when loading checkpoints",
-        )
-        parser.add_argument(
-            "--untie-weights-roberta",
-            action="store_true",
-            help="Untie weights between embeddings and classifiers in RoBERTa",
-        )
-        # args for "Reducing Transformer Depth on Demand with Structured Dropout" (Fan et al., 2019)
-        parser.add_argument(
-            "--encoder-layerdrop",
-            type=float,
-            metavar="D",
-            default=0,
-            help="LayerDrop probability for encoder",
-        )
-        parser.add_argument(
-            "--encoder-layers-to-keep",
-            default=None,
-            help="which layers to *keep* when pruning as a comma-separated list",
-        )
-        # args for Training with Quantization Noise for Extreme Model Compression ({Fan*, Stock*} et al., 2020)
-        parser.add_argument(
-            "--quant-noise-pq",
-            type=float,
-            metavar="D",
-            default=0,
-            help="iterative PQ quantization noise at training time",
-        )
-        parser.add_argument(
-            "--quant-noise-pq-block-size",
-            type=int,
-            metavar="D",
-            default=8,
-            help="block size of quantization noise at training time",
-        )
-        parser.add_argument(
-            "--quant-noise-scalar",
-            type=float,
-            metavar="D",
-            default=0,
-            help="scalar quantization noise and scalar quantization at training time",
-        )
-        # args for "Better Fine-Tuning by Reducing Representational Collapse" (Aghajanyan et al. 2020)
-        parser.add_argument(
-            "--spectral-norm-classification-head",
-            action="store_true",
-            default=False,
-            help="Apply spectral normalization on the classification head",
-        )
-        # args for Fully Sharded Data Parallel (FSDP) training
-        parser.add_argument(
-            "--min-params-to-wrap",
-            type=int,
-            metavar="D",
-            default=DEFAULT_MIN_PARAMS_TO_WRAP,
-            help=(
-                "minimum number of params for a layer to be wrapped with FSDP() when "
-                "training with --ddp-backend=fully_sharded. Smaller values will "
-                "improve memory efficiency, but may make torch.distributed "
-                "communication less efficient due to smaller input sizes. This option "
-                "is set to 0 (i.e., always wrap) when --checkpoint-activations or "
-                "--offload-activations are passed."
-            ),
-        )
-        # args for AdaPruning
-        # In short, it adds regularizarion for the multihead attention module and feed forward neural nets
-        # For more details, please refer to the paper https://openreview.net/forum?id=_CMSV7FTzGI
-        parser.add_argument(
-            "--mha-reg-scale-factor",
-            type=float,
-            metavar="D",
-            default=0.0,
-            help="scaling factor for regularization term in adptive pruning, recommendation is 0.000375",
-        )
-        parser.add_argument(
-            "--ffn-reg-scale-factor",
-            type=float,
-            metavar="D",
-            default=0.0,
-            help="scaling factor for regularization term in adptive pruning, recommendation is 0.000375",
-        )
-        parser.add_argument(
-            "--mha-heads-to-keep",
-            type=int,
-            metavar="D",
-            default=-1,
-            help="number of heads to keep in each multi-head attention module, -1 means keeping all heads",
-        )
-        parser.add_argument(
-            "--ffn-blocks-to-remove",
-            type=int,
-            metavar="D",
-            default=-1,
-            help="number of feedforward blocks to remove in each transformer layer, -1 means keeping all ffn blocks",
-        )
-
-    @classmethod
-    def build_model(cls, args, task):
-        """Build a new model instance."""
-
-        from omegaconf import OmegaConf
-
-        if OmegaConf.is_config(args):
-            OmegaConf.set_struct(args, False)
-
-        # make sure all arguments are present
-        base_architecture(args)
-
-        if not safe_hasattr(args, "max_positions"):
-            if not safe_hasattr(args, "tokens_per_sample"):
-                args.tokens_per_sample = task.max_positions()
-            args.max_positions = args.tokens_per_sample
-
-        encoder = RobertaEncoder(args, task.source_dictionary)
-
-        if OmegaConf.is_config(args):
-            OmegaConf.set_struct(args, True)
-
-        return cls(args, encoder)
-
-    def forward(
-        self,
-        src_tokens,
-        features_only=False,
-        return_all_hiddens=False,
-        classification_head_name=None,
-        **kwargs,
-    ):
-        if classification_head_name is not None:
-            features_only = True
-
-        x, extra = self.encoder(src_tokens, features_only, return_all_hiddens, **kwargs)
-
-        if classification_head_name is not None:
-            x = self.classification_heads[classification_head_name](x)
-        return x, extra
-
-    def _get_adaptive_head_loss(self):
-        norm_loss = 0
-        scaling = float(self.args.mha_reg_scale_factor)
-        for layer in self.encoder.sentence_encoder.layers:
-            norm_loss_layer = 0
-            for i in range(layer.self_attn.num_heads):
-                start_idx = i * layer.self_attn.head_dim
-                end_idx = (i + 1) * layer.self_attn.head_dim
-                norm_loss_layer += scaling * (
-                    torch.sum(
-                        torch.abs(
-                            layer.self_attn.q_proj.weight[
-                                start_idx:end_idx,
-                            ]
-                        )
-                    )
-                    + torch.sum(
-                        torch.abs(layer.self_attn.q_proj.bias[start_idx:end_idx])
-                    )
-                )
-                norm_loss_layer += scaling * (
-                    torch.sum(
-                        torch.abs(
-                            layer.self_attn.k_proj.weight[
-                                start_idx:end_idx,
-                            ]
-                        )
-                    )
-                    + torch.sum(
-                        torch.abs(layer.self_attn.k_proj.bias[start_idx:end_idx])
-                    )
-                )
-                norm_loss_layer += scaling * (
-                    torch.sum(
-                        torch.abs(
-                            layer.self_attn.v_proj.weight[
-                                start_idx:end_idx,
-                            ]
-                        )
-                    )
-                    + torch.sum(
-                        torch.abs(layer.self_attn.v_proj.bias[start_idx:end_idx])
-                    )
-                )
-
-            norm_loss += norm_loss_layer
-        return norm_loss
-
-    def _get_adaptive_ffn_loss(self):
-        ffn_scale_factor = float(self.args.ffn_reg_scale_factor)
-        filter_loss = 0
-        for layer in self.encoder.sentence_encoder.layers:
-            filter_loss += torch.sum(
-                torch.abs(layer.fc1.weight * ffn_scale_factor)
-            ) + torch.sum(torch.abs(layer.fc2.weight * ffn_scale_factor))
-            filter_loss += torch.sum(
-                torch.abs(layer.fc1.bias * ffn_scale_factor)
-            ) + torch.sum(torch.abs(layer.fc2.bias * ffn_scale_factor))
-        return filter_loss
-
-    def get_normalized_probs(self, net_output, log_probs, sample=None):
-        """Get normalized probabilities (or log probs) from a net's output."""
-        logits = net_output[0].float()
-        if log_probs:
-            return F.log_softmax(logits, dim=-1)
-        else:
-            return F.softmax(logits, dim=-1)
-
-    def register_classification_head(
-        self, name, num_classes=None, inner_dim=None, **kwargs
-    ):
-        """Register a classification head."""
-        if name in self.classification_heads:
-            prev_num_classes = self.classification_heads[name].out_proj.out_features
-            prev_inner_dim = self.classification_heads[name].dense.out_features
-            if num_classes != prev_num_classes or inner_dim != prev_inner_dim:
-                logger.warning(
-                    're-registering head "{}" with num_classes {} (prev: {}) '
-                    "and inner_dim {} (prev: {})".format(
-                        name, num_classes, prev_num_classes, inner_dim, prev_inner_dim
-                    )
-                )
-        self.classification_heads[name] = RobertaClassificationHead(
-            input_dim=self.args.encoder_embed_dim,
-            inner_dim=inner_dim or self.args.encoder_embed_dim,
-            num_classes=num_classes,
-            activation_fn=self.args.pooler_activation_fn,
-            pooler_dropout=self.args.pooler_dropout,
-            q_noise=self.args.quant_noise_pq,
-            qn_block_size=self.args.quant_noise_pq_block_size,
-            do_spectral_norm=self.args.spectral_norm_classification_head,
-        )
-
-    @property
-    def supported_targets(self):
-        return {"self"}
-
-    @classmethod
-    def from_pretrained(
-        cls,
-        model_name_or_path,
-        checkpoint_file="model.pt",
-        data_name_or_path=".",
-        bpe="gpt2",
-        **kwargs,
-    ):
-        from fairseq import hub_utils
-
-        x = hub_utils.from_pretrained(
-            model_name_or_path,
-            checkpoint_file,
-            data_name_or_path,
-            archive_map=cls.hub_models(),
-            bpe=bpe,
-            load_checkpoint_heads=True,
-            **kwargs,
-        )
-
-        logger.info(x["args"])
-        return RobertaHubInterface(x["args"], x["task"], x["models"][0])
-
-    def upgrade_state_dict_named(self, state_dict, name):
-        prefix = name + "." if name != "" else ""
-
-        # rename decoder -> encoder before upgrading children modules
-        for k in list(state_dict.keys()):
-            if k.startswith(prefix + "decoder"):
-                new_k = prefix + "encoder" + k[len(prefix + "decoder") :]
-                state_dict[new_k] = state_dict[k]
-                del state_dict[k]
-
-        # rename emb_layer_norm -> layernorm_embedding
-        for k in list(state_dict.keys()):
-            if ".emb_layer_norm." in k:
-                new_k = k.replace(".emb_layer_norm.", ".layernorm_embedding.")
-                state_dict[new_k] = state_dict[k]
-                del state_dict[k]
-
-        # upgrade children modules
-        super().upgrade_state_dict_named(state_dict, name)
-
-        # Handle new classification heads present in the state dict.
-        current_head_names = (
-            []
-            if not hasattr(self, "classification_heads")
-            else self.classification_heads.keys()
-        )
-        keys_to_delete = []
-        for k in state_dict.keys():
-            if not k.startswith(prefix + "classification_heads."):
-                continue
-
-            head_name = k[len(prefix + "classification_heads.") :].split(".")[0]
-            num_classes = state_dict[
-                prefix + "classification_heads." + head_name + ".out_proj.weight"
-            ].size(0)
-            inner_dim = state_dict[
-                prefix + "classification_heads." + head_name + ".dense.weight"
-            ].size(0)
-
-            if getattr(self.args, "load_checkpoint_heads", False):
-                if head_name not in current_head_names:
-                    self.register_classification_head(head_name, num_classes, inner_dim)
-            else:
-                if head_name not in current_head_names:
-                    logger.warning(
-                        "deleting classification head ({}) from checkpoint "
-                        "not present in current model: {}".format(head_name, k)
-                    )
-                    keys_to_delete.append(k)
-                elif (
-                    num_classes
-                    != self.classification_heads[head_name].out_proj.out_features
-                    or inner_dim
-                    != self.classification_heads[head_name].dense.out_features
-                ):
-                    logger.warning(
-                        "deleting classification head ({}) from checkpoint "
-                        "with different dimensions than current model: {}".format(
-                            head_name, k
-                        )
-                    )
-                    keys_to_delete.append(k)
-        for k in keys_to_delete:
-            del state_dict[k]
-
-        # Copy any newly-added classification heads into the state dict
-        # with their current weights.
-        if hasattr(self, "classification_heads"):
-            cur_state = self.classification_heads.state_dict()
-            for k, v in cur_state.items():
-                if prefix + "classification_heads." + k not in state_dict:
-                    logger.info("Overwriting " + prefix + "classification_heads." + k)
-                    state_dict[prefix + "classification_heads." + k] = v
-
-            # adapt data2vec models
-            if (
-                "encoder._ema" in state_dict
-                and "encoder.lm_head.weight" not in state_dict
-            ):
-                lm_state = self.encoder.lm_head.state_dict()
-                for k, v in lm_state.items():
-                    state_dict["encoder.lm_head." + k] = v
-
-            for k in list(state_dict.keys()):
-                if k.startswith("encoder.regression_head") or k == "encoder._ema":
-                    del state_dict[k]
-
-
-class RobertaLMHead(nn.Module):
-    """Head for masked language modeling."""
-
-    def __init__(self, embed_dim, output_dim, activation_fn, weight=None):
-        super().__init__()
-        self.dense = nn.Linear(embed_dim, embed_dim)
-        self.activation_fn = utils.get_activation_fn(activation_fn)
-        self.layer_norm = LayerNorm(embed_dim)
-
-        if weight is None:
-            weight = nn.Linear(embed_dim, output_dim, bias=False).weight
-        self.weight = weight
-        self.bias = nn.Parameter(torch.zeros(output_dim))
-
-    def forward(self, features, masked_tokens=None, **kwargs):
-        # Only project the masked tokens while training,
-        # saves both memory and computation
-        if masked_tokens is not None:
-            features = features[masked_tokens, :]
-
-        x = self.dense(features)
-        x = self.activation_fn(x)
-        x = self.layer_norm(x)
-        # project back to size of vocabulary with bias
-        x = F.linear(x, self.weight) + self.bias
-        return x
-
-
-class RobertaClassificationHead(nn.Module):
-    """Head for sentence-level classification tasks."""
-
-    def __init__(
-        self,
-        input_dim,
-        inner_dim,
-        num_classes,
-        activation_fn,
-        pooler_dropout,
-        q_noise=0,
-        qn_block_size=8,
-        do_spectral_norm=False,
-    ):
-        super().__init__()
-        self.dense = nn.Linear(input_dim, inner_dim)
-        self.activation_fn = utils.get_activation_fn(activation_fn)
-        self.dropout = nn.Dropout(p=pooler_dropout)
-        self.out_proj = apply_quant_noise_(
-            nn.Linear(inner_dim, num_classes), q_noise, qn_block_size
-        )
-        if do_spectral_norm:
-            if q_noise != 0:
-                raise NotImplementedError(
-                    "Attempting to use Spectral Normalization with Quant Noise. This is not officially supported"
-                )
-            self.out_proj = torch.nn.utils.spectral_norm(self.out_proj)
-
-    def forward(self, features, **kwargs):
-        x = features[:, 0, :]  # take <s> token (equiv. to [CLS])
-        x = self.dropout(x)
-        x = self.dense(x)
-        x = self.activation_fn(x)
-        x = self.dropout(x)
-        x = self.out_proj(x)
-        return x
-
-
-class RobertaEncoder(FairseqEncoder):
-    """RoBERTa encoder."""
-
-    def __init__(self, args, dictionary):
-        super().__init__(dictionary)
-
-        # set any missing default values
-        base_architecture(args)
-        self.args = args
-
-        if args.encoder_layers_to_keep:
-            args.encoder_layers = len(args.encoder_layers_to_keep.split(","))
-
-        embed_tokens = self.build_embedding(
-            len(dictionary), args.encoder_embed_dim, dictionary.pad()
-        )
-
-        self.sentence_encoder = self.build_encoder(args, dictionary, embed_tokens)
-
-        self.lm_head = self.build_lm_head(
-            embed_dim=args.encoder_embed_dim,
-            output_dim=len(dictionary),
-            activation_fn=args.activation_fn,
-            weight=(
-                self.sentence_encoder.embed_tokens.weight
-                if not args.untie_weights_roberta
-                else None
-            ),
-        )
-
-    def build_embedding(self, vocab_size, embedding_dim, padding_idx):
-        return nn.Embedding(vocab_size, embedding_dim, padding_idx)
-
-    def build_encoder(self, args, dictionary, embed_tokens):
-        encoder = TransformerEncoder(args, dictionary, embed_tokens)
-        encoder.apply(init_bert_params)
-        return encoder
-
-    def build_lm_head(self, embed_dim, output_dim, activation_fn, weight):
-        return RobertaLMHead(embed_dim, output_dim, activation_fn, weight)
-
-    def forward(
-        self,
-        src_tokens,
-        features_only=False,
-        return_all_hiddens=False,
-        masked_tokens=None,
-        **unused,
-    ):
-        """
-        Args:
-            src_tokens (LongTensor): input tokens of shape `(batch, src_len)`
-            features_only (bool, optional): skip LM head and just return
-                features. If True, the output will be of shape
-                `(batch, src_len, embed_dim)`.
-            return_all_hiddens (bool, optional): also return all of the
-                intermediate hidden states (default: False).
-
-        Returns:
-            tuple:
-                - the LM output of shape `(batch, src_len, vocab)`
-                - a dictionary of additional data, where 'inner_states'
-                  is a list of hidden states. Note that the hidden
-                  states have shape `(src_len, batch, vocab)`.
-        """
-        x, extra = self.extract_features(
-            src_tokens, return_all_hiddens=return_all_hiddens
-        )
-        if not features_only:
-            x = self.output_layer(x, masked_tokens=masked_tokens)
-        return x, extra
-
-    def extract_features(self, src_tokens, return_all_hiddens=False, **kwargs):
-        encoder_out = self.sentence_encoder(
-            src_tokens,
-            return_all_hiddens=return_all_hiddens,
-            token_embeddings=kwargs.get("token_embeddings", None),
-        )
-        # T x B x C -> B x T x C
-        features = encoder_out["encoder_out"][0].transpose(0, 1)
-        inner_states = encoder_out["encoder_states"] if return_all_hiddens else None
-        return features, {"inner_states": inner_states}
-
-    def output_layer(self, features, masked_tokens=None, **unused):
-        return self.lm_head(features, masked_tokens)
-
-    def max_positions(self):
-        """Maximum output length supported by the encoder."""
-        return self.args.max_positions
-
-
-@register_model_architecture("roberta", "roberta")
-def base_architecture(args):
-    args.encoder_layers = safe_getattr(args, "encoder_layers", 12)
-    args.encoder_embed_dim = safe_getattr(args, "encoder_embed_dim", 768)
-    args.encoder_ffn_embed_dim = safe_getattr(args, "encoder_ffn_embed_dim", 3072)
-    args.encoder_attention_heads = safe_getattr(args, "encoder_attention_heads", 12)
-
-    args.dropout = safe_getattr(args, "dropout", 0.1)
-    args.attention_dropout = safe_getattr(args, "attention_dropout", 0.1)
-    args.activation_dropout = safe_getattr(args, "activation_dropout", 0.0)
-    args.pooler_dropout = safe_getattr(args, "pooler_dropout", 0.0)
-
-    args.max_source_positions = safe_getattr(args, "max_positions", 512)
-    args.no_token_positional_embeddings = safe_getattr(
-        args, "no_token_positional_embeddings", False
-    )
-
-    # BERT has a few structural differences compared to the original Transformer
-    args.encoder_learned_pos = safe_getattr(args, "encoder_learned_pos", True)
-    args.layernorm_embedding = safe_getattr(args, "layernorm_embedding", True)
-    args.no_scale_embedding = safe_getattr(args, "no_scale_embedding", True)
-    args.activation_fn = safe_getattr(args, "activation_fn", "gelu")
-    args.encoder_normalize_before = safe_getattr(
-        args, "encoder_normalize_before", False
-    )
-    args.pooler_activation_fn = safe_getattr(args, "pooler_activation_fn", "tanh")
-    args.untie_weights_roberta = safe_getattr(args, "untie_weights_roberta", False)
-
-    # Adaptive input config
-    args.adaptive_input = safe_getattr(args, "adaptive_input", False)
-
-    # LayerDrop config
-    args.encoder_layerdrop = safe_getattr(args, "encoder_layerdrop", 0.0)
-    args.encoder_layers_to_keep = safe_getattr(args, "encoder_layers_to_keep", None)
-
-    # Quantization noise config
-    args.quant_noise_pq = safe_getattr(args, "quant_noise_pq", 0)
-    args.quant_noise_pq_block_size = safe_getattr(args, "quant_noise_pq_block_size", 8)
-    args.quant_noise_scalar = safe_getattr(args, "quant_noise_scalar", 0)
-
-    # R4F config
-    args.spectral_norm_classification_head = safe_getattr(
-        args, "spectral_norm_classification_head", False
-    )
-
-
-@register_model_architecture("roberta", "roberta_prenorm")
-def roberta_prenorm_architecture(args):
-    args.layernorm_embedding = safe_getattr(args, "layernorm_embedding", False)
-    args.encoder_normalize_before = safe_getattr(args, "encoder_normalize_before", True)
-    base_architecture(args)
-
-
-@register_model_architecture("roberta", "roberta_base")
-def roberta_base_architecture(args):
-    base_architecture(args)
-
-
-@register_model_architecture("roberta", "roberta_large")
-def roberta_large_architecture(args):
-    args.encoder_layers = safe_getattr(args, "encoder_layers", 24)
-    args.encoder_embed_dim = safe_getattr(args, "encoder_embed_dim", 1024)
-    args.encoder_ffn_embed_dim = safe_getattr(args, "encoder_ffn_embed_dim", 4096)
-    args.encoder_attention_heads = safe_getattr(args, "encoder_attention_heads", 16)
-    base_architecture(args)
-
-
-@register_model_architecture("roberta", "xlm")
-def xlm_architecture(args):
-    args.encoder_layers = safe_getattr(args, "encoder_layers", 16)
-    args.encoder_embed_dim = safe_getattr(args, "encoder_embed_dim", 1280)
-    args.encoder_ffn_embed_dim = safe_getattr(args, "encoder_ffn_embed_dim", 1280 * 4)
-    args.encoder_attention_heads = safe_getattr(args, "encoder_attention_heads", 16)
-    base_architecture(args)

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/roberta/model_camembert.py

@@ -1,50 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-"""
-CamemBERT: a Tasty French Language Model
-"""
-
-from fairseq.models import register_model
-
-from .hub_interface import RobertaHubInterface
-from .model import RobertaModel
-
-
-@register_model("camembert")
-class CamembertModel(RobertaModel):
-    @classmethod
-    def hub_models(cls):
-        return {
-            "camembert": "http://dl.fbaipublicfiles.com/fairseq/models/camembert-base.tar.gz",
-            "camembert.v0": "http://dl.fbaipublicfiles.com/fairseq/models/camembert-base.tar.gz",
-            "camembert-base": "http://dl.fbaipublicfiles.com/fairseq/models/camembert-base.tar.gz",
-            "camembert-large": "http://dl.fbaipublicfiles.com/fairseq/models/camembert-large.tar.gz",
-            "camembert-base-ccnet": "http://dl.fbaipublicfiles.com/fairseq/models/camembert-base-ccnet.tar.gz",
-            "camembert-base-ccnet-4gb": "http://dl.fbaipublicfiles.com/fairseq/models/camembert-base-ccnet-4gb.tar.gz",
-            "camembert-base-wikipedia-4gb": "http://dl.fbaipublicfiles.com/fairseq/models/camembert-base-wikipedia-4gb.tar.gz",
-            "camembert-base-oscar-4gb": "http://dl.fbaipublicfiles.com/fairseq/models/camembert-base-oscar-4gb.tar.gz",
-        }
-
-    @classmethod
-    def from_pretrained(
-        cls,
-        model_name_or_path,
-        checkpoint_file="model.pt",
-        data_name_or_path=".",
-        bpe="sentencepiece",
-        **kwargs
-    ):
-        from fairseq import hub_utils
-
-        x = hub_utils.from_pretrained(
-            model_name_or_path,
-            checkpoint_file,
-            data_name_or_path,
-            archive_map=cls.hub_models(),
-            bpe=bpe,
-            load_checkpoint_heads=True,
-            **kwargs,
-        )
-        return RobertaHubInterface(x["args"], x["task"], x["models"][0])

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/roberta/model_gottbert.py

@@ -1,49 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-"""
-GottBERT: a pure German Language Model
-"""
-
-from fairseq.models import register_model
-
-from .hub_interface import RobertaHubInterface
-from .model import RobertaModel
-
-
-@register_model("gottbert")
-class GottbertModel(RobertaModel):
-    @classmethod
-    def hub_models(cls):
-        return {
-            "gottbert-base": "https://dl.gottbert.de/fairseq/models/gottbert-base.tar.gz",
-        }
-
-    @classmethod
-    def from_pretrained(
-        cls,
-        model_name_or_path,
-        checkpoint_file="model.pt",
-        data_name_or_path=".",
-        bpe="hf_byte_bpe",
-        bpe_vocab="vocab.json",
-        bpe_merges="merges.txt",
-        bpe_add_prefix_space=False,
-        **kwargs
-    ):
-        from fairseq import hub_utils
-
-        x = hub_utils.from_pretrained(
-            model_name_or_path,
-            checkpoint_file,
-            data_name_or_path,
-            archive_map=cls.hub_models(),
-            bpe=bpe,
-            load_checkpoint_heads=True,
-            bpe_vocab=bpe_vocab,
-            bpe_merges=bpe_merges,
-            bpe_add_prefix_space=bpe_add_prefix_space,
-            **kwargs,
-        )
-        return RobertaHubInterface(x["args"], x["task"], x["models"][0])

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/roberta/model_xlmr.py

@@ -1,46 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-"""
-Unsupervised Cross-lingual Representation Learning at Scale
-"""
-
-from fairseq.models import register_model
-
-from .hub_interface import RobertaHubInterface
-from .model import RobertaModel
-
-
-@register_model("xlmr")
-class XLMRModel(RobertaModel):
-    @classmethod
-    def hub_models(cls):
-        return {
-            "xlmr.base": "http://dl.fbaipublicfiles.com/fairseq/models/xlmr.base.tar.gz",
-            "xlmr.large": "http://dl.fbaipublicfiles.com/fairseq/models/xlmr.large.tar.gz",
-            "xlmr.xl": "http://dl.fbaipublicfiles.com/fairseq/models/xlmr/xlmr.xl.tar.gz",
-            "xlmr.xxl": "http://dl.fbaipublicfiles.com/fairseq/models/xlmr/xlmr.xxl.tar.gz",
-        }
-
-    @classmethod
-    def from_pretrained(
-        cls,
-        model_name_or_path,
-        checkpoint_file="model.pt",
-        data_name_or_path=".",
-        bpe="sentencepiece",
-        **kwargs
-    ):
-        from fairseq import hub_utils
-
-        x = hub_utils.from_pretrained(
-            model_name_or_path,
-            checkpoint_file,
-            data_name_or_path,
-            archive_map=cls.hub_models(),
-            bpe=bpe,
-            load_checkpoint_heads=True,
-            **kwargs,
-        )
-        return RobertaHubInterface(x["args"], x["task"], x["models"][0])

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/speech_to_speech/__init__.py

@@ -1,8 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-from .modules import *  # noqa
-from .s2s_conformer import *  # noqa
-from .s2s_transformer import *  # noqa

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/speech_to_speech/modules.py

@@ -1,58 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-import torch
-from fairseq.models import FairseqEncoder
-from fairseq.models.transformer import Linear
-from torch import nn
-
-
-class CTCDecoder(FairseqEncoder):
-    def __init__(self, dictionary, in_dim):
-        super().__init__(dictionary)
-        self.proj = nn.Linear(in_dim, len(dictionary))
-
-    def forward(self, src_tokens, src_lengths=None, **kwargs):
-        encoder_out = self.proj(src_tokens)
-        return {"encoder_out": encoder_out}
-
-
-class StackedEmbedding(nn.Embedding):
-    """Embedding module that supports stacked units -> single embedding"""
-
-    def __init__(self, num_embeddings, embed_dim, padding_idx, num_stacked=1):
-        super().__init__(num_embeddings, embed_dim, padding_idx)
-        # follow transformer.Embedding
-        nn.init.normal_(self.weight, mean=0, std=embed_dim**-0.5)
-        nn.init.constant_(self.weight[padding_idx], 0)
-
-        self.offset = (
-            4  # skip <bos>, <pad>, <eos>, <unk>, specific to fairseq dictionary
-        )
-        self.vocab_size = num_embeddings - self.offset
-        self.num_stacked = num_stacked
-
-        if self.num_stacked > 1:
-            self.project_in_dim = Linear(embed_dim * num_stacked, embed_dim, bias=False)
-
-    def forward(self, input):
-        if self.num_stacked == 1:
-            return super().forward(input)
-
-        # expand input indices
-        mask = input >= self.offset
-        stacked_input = []
-        cum_input = input.new_zeros(input.shape)
-        for i in range(1, self.num_stacked + 1):
-            div = pow(self.vocab_size, i)
-            next_input = torch.remainder(input - self.offset - cum_input, div)
-            cum_input += next_input
-            next_input = torch.floor_divide(next_input, div // self.vocab_size)
-            stacked_input.append((next_input + self.offset) * mask + input * ~mask)
-
-        stacked_input = torch.stack(stacked_input[::-1], dim=2)
-        embed = super().forward(stacked_input).view(input.size(0), input.size(1), -1)
-        embed = self.project_in_dim(embed)
-        return embed

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/speech_to_speech/s2s_conformer.py

@@ -1,112 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-import logging
-from pathlib import Path
-
-import torch
-from fairseq import checkpoint_utils
-from fairseq.data.audio.data_cfg import S2SDataConfig
-from fairseq.models import (
-    register_model,
-    register_model_architecture,
-)
-from fairseq.models.speech_to_speech import (
-    S2UTTransformerModel,
-)
-from fairseq.models.speech_to_speech import (
-    s2ut_architecture_base as s2ut_transformer_architecture_base,
-)
-from fairseq.models.speech_to_text import S2TConformerEncoder
-from fairseq.models.transformer import (
-    Linear,
-)
-
-logger = logging.getLogger(__name__)
-
-
-class S2SConformerEncoder(S2TConformerEncoder):
-    """Based on S2T transformer encoder, with support
-    to incorporate target speaker embedding."""
-
-    def __init__(self, args):
-        super().__init__(args)
-
-        self.spk_emb_proj = None
-        if args.target_speaker_embed:
-            self.spk_emb_proj = Linear(
-                args.encoder_embed_dim + args.speaker_embed_dim, args.encoder_embed_dim
-            )
-
-    def forward(
-        self, src_tokens, src_lengths, tgt_speaker=None, return_all_hiddens=False
-    ):
-        out = super().forward(src_tokens, src_lengths, return_all_hiddens)
-
-        if self.spk_emb_proj:
-            x = out["encoder_out"][0]
-            seq_len, bsz, _ = x.size()
-            tgt_speaker_emb = tgt_speaker.view(1, bsz, -1).expand(seq_len, bsz, -1)
-            x = self.spk_emb_proj(torch.cat([x, tgt_speaker_emb], dim=2))
-            out["encoder_out"][0] = x
-
-        return out
-
-
-@register_model("s2ut_conformer")
-class S2UTConformerModel(S2UTTransformerModel):
-    """
-    Direct speech-to-speech translation model with S2T Conformer encoder + Transformer discrete unit decoder
-    """
-
-    @staticmethod
-    def add_args(parser):
-        S2UTTransformerModel.add_args(parser)
-        parser.add_argument("--depthwise-conv-kernel-size", default=31)
-        parser.add_argument(
-            "--attn-type",
-            default=None,
-            help="If not specified uses fairseq MHA. Other valid option is espnet for using conformer",
-        )
-        parser.add_argument(
-            "--pos-enc-type",
-            default="abs",
-            help="Must be specified in addition to attn-type=espnet for rel_pos and rope",
-        )
-
-    @classmethod
-    def build_encoder(cls, args):
-        print(args)
-        data_cfg = S2SDataConfig(Path(args.data) / args.config_yaml)
-        args.input_feat_per_channel = data_cfg.input_feat_per_channel
-        args.input_channels = data_cfg.input_transformed_channels
-
-        encoder = S2SConformerEncoder(args)
-        pretraining_path = getattr(args, "load_pretrained_encoder_from", None)
-        if pretraining_path is not None:
-            if not Path(pretraining_path).exists():
-                logger.warning(
-                    f"skipped pretraining because {pretraining_path} does not exist"
-                )
-            else:
-                encoder = checkpoint_utils.load_pretrained_component_from_model(
-                    component=encoder, checkpoint=pretraining_path
-                )
-                logger.info(f"loaded pretrained encoder from: {pretraining_path}")
-        return encoder
-
-
-@register_model_architecture("s2ut_conformer", "s2ut_conformer")
-def s2ut_base_architecture(args):
-    args.attn_type = getattr(args, "attn_type", None)
-    args.pos_enc_type = getattr(args, "pos_enc_type", "abs")
-    args.max_source_positions = getattr(args, "max_source_positions", 6000)
-    args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 256)
-    args.encoder_ffn_embed_dim = getattr(args, "encoder_ffn_embed_dim", 2048)
-    args.encoder_attention_heads = getattr(args, "encoder_attention_heads", 4)
-    args.dropout = getattr(args, "dropout", 0.1)
-    args.encoder_layers = getattr(args, "encoder_layers", 16)
-    args.depthwise_conv_kernel_size = getattr(args, "depthwise_conv_kernel_size", 31)
-    s2ut_transformer_architecture_base(args)

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/speech_to_speech/s2s_transformer.py

@@ -1,701 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-import logging
-from pathlib import Path
-from typing import Any, Dict, List, Optional
-
-import torch
-from fairseq import checkpoint_utils, utils
-from fairseq.models import (
-    FairseqEncoderDecoderModel,
-    FairseqEncoderModel,
-    FairseqLanguageModel,
-    register_model,
-    register_model_architecture,
-)
-from fairseq.models.speech_to_speech.modules import CTCDecoder, StackedEmbedding
-from fairseq.models.speech_to_text import S2TTransformerEncoder
-from fairseq.models.text_to_speech import TTSTransformerDecoder
-from fairseq.models.transformer import (
-    Linear,
-    TransformerDecoder,
-    TransformerModelBase,
-)
-from torch import Tensor
-
-logger = logging.getLogger(__name__)
-
-
-class S2STransformerEncoder(S2TTransformerEncoder):
-    """Based on S2T transformer encoder, with support
-    to incorporate target speaker embedding."""
-
-    def __init__(self, args):
-        super().__init__(args)
-
-        self.spk_emb_proj = None
-        if args.target_speaker_embed:
-            self.spk_emb_proj = Linear(
-                args.encoder_embed_dim + args.speaker_embed_dim, args.encoder_embed_dim
-            )
-
-    def forward(
-        self, src_tokens, src_lengths, tgt_speaker=None, return_all_hiddens=False
-    ):
-        out = super().forward(src_tokens, src_lengths, return_all_hiddens)
-
-        if self.spk_emb_proj:
-            x = out["encoder_out"][0]
-            seq_len, bsz, _ = x.size()
-            tgt_speaker_emb = tgt_speaker.view(1, bsz, -1).expand(seq_len, bsz, -1)
-            x = self.spk_emb_proj(torch.cat([x, tgt_speaker_emb], dim=2))
-            out["encoder_out"][0] = x
-
-        return out
-
-
-class TransformerUnitDecoder(TransformerDecoder):
-    """Based on Transformer decoder, with support to decoding stacked units"""
-
-    def __init__(
-        self,
-        args,
-        dictionary,
-        embed_tokens,
-        no_encoder_attn=False,
-        output_projection=None,
-    ):
-        super().__init__(
-            args, dictionary, embed_tokens, no_encoder_attn, output_projection
-        )
-        self.n_frames_per_step = args.n_frames_per_step
-
-        self.out_proj_n_frames = (
-            Linear(
-                self.output_embed_dim,
-                self.output_embed_dim * self.n_frames_per_step,
-                bias=False,
-            )
-            if self.n_frames_per_step > 1
-            else None
-        )
-
-    def forward(
-        self,
-        prev_output_tokens,
-        encoder_out: Optional[Dict[str, List[Tensor]]] = None,
-        incremental_state: Optional[Dict[str, Dict[str, Optional[Tensor]]]] = None,
-        features_only: bool = False,
-        full_context_alignment: bool = False,
-        alignment_layer: Optional[int] = None,
-        alignment_heads: Optional[int] = None,
-        src_lengths: Optional[Any] = None,
-        return_all_hiddens: bool = False,
-    ):
-        """
-        Args:
-            prev_output_tokens (LongTensor): previous decoder outputs of shape
-                `(batch, tgt_len)`, for teacher forcing
-            encoder_out (optional): output from the encoder, used for
-                encoder-side attention, should be of size T x B x C
-            incremental_state (dict): dictionary used for storing state during
-                :ref:`Incremental decoding`
-            features_only (bool, optional): only return features without
-                applying output layer (default: False).
-            full_context_alignment (bool, optional): don't apply
-                auto-regressive mask to self-attention (default: False).
-
-        Returns:
-            tuple:
-                - the decoder's output of shape `(batch, tgt_len, vocab)`
-                - a dictionary with any model-specific outputs
-        """
-
-        x, extra = self.extract_features(
-            prev_output_tokens,
-            encoder_out=encoder_out,
-            incremental_state=incremental_state,
-            full_context_alignment=full_context_alignment,
-            alignment_layer=alignment_layer,
-            alignment_heads=alignment_heads,
-        )
-
-        if not features_only:
-            bsz, seq_len, d = x.size()
-            if self.out_proj_n_frames:
-                x = self.out_proj_n_frames(x)
-            x = self.output_layer(x.view(bsz, seq_len, self.n_frames_per_step, d))
-            x = x.view(bsz, seq_len * self.n_frames_per_step, -1)
-            if (
-                incremental_state is None and self.n_frames_per_step > 1
-            ):  # teacher-forcing mode in training
-                x = x[
-                    :, : -(self.n_frames_per_step - 1), :
-                ]  # remove extra frames after <eos>
-
-        return x, extra
-
-    def upgrade_state_dict_named(self, state_dict, name):
-        if self.n_frames_per_step > 1:
-            move_keys = [
-                (
-                    f"{name}.project_in_dim.weight",
-                    f"{name}.embed_tokens.project_in_dim.weight",
-                )
-            ]
-            for from_k, to_k in move_keys:
-                if from_k in state_dict and to_k not in state_dict:
-                    state_dict[to_k] = state_dict[from_k]
-                    del state_dict[from_k]
-
-
-class S2STransformerMultitaskModelBase(FairseqEncoderDecoderModel):
-    @classmethod
-    def build_encoder(cls, args):
-        encoder = S2STransformerEncoder(args)
-        pretraining_path = getattr(args, "load_pretrained_encoder_from", None)
-        if pretraining_path is not None:
-            if not Path(pretraining_path).exists():
-                logger.warning(
-                    f"skipped pretraining because {pretraining_path} does not exist"
-                )
-            else:
-                encoder = checkpoint_utils.load_pretrained_component_from_model(
-                    component=encoder, checkpoint=pretraining_path
-                )
-                logger.info(f"loaded pretrained encoder from: {pretraining_path}")
-        return encoder
-
-    @classmethod
-    def build_multitask_decoder(cls, args, tgt_dict, in_dim):
-        decoder_args = args.decoder_args
-        decoder_args.encoder_embed_dim = in_dim
-        if args.decoder_type == "transformer":
-            base_multitask_text_transformer_decoder_arch(decoder_args)
-            task_decoder = TransformerDecoder(
-                decoder_args,
-                tgt_dict,
-                embed_tokens=TransformerModelBase.build_embedding(
-                    decoder_args,
-                    tgt_dict,
-                    decoder_args.decoder_embed_dim,
-                ),
-            )
-        elif args.decoder_type == "ctc":
-            task_decoder = CTCDecoder(
-                dictionary=tgt_dict,
-                in_dim=in_dim,
-            )
-        else:
-            raise NotImplementedError(
-                "currently only support multitask decoder_type 'transformer', 'ctc'"
-            )
-
-        return task_decoder
-
-    @classmethod
-    def build_model(cls, args, task):
-        encoder = cls.build_encoder(args)
-        decoder = (
-            cls.build_decoder(args, task.target_dictionary)
-            if task.args.target_is_code
-            else cls.build_decoder(args)
-        )
-        base_model = cls(encoder, decoder)
-
-        # set up multitask decoders
-        base_model.multitask_decoders = {}
-        for task_name, task_obj in task.multitask_tasks.items():
-            in_dim = (
-                args.encoder_embed_dim
-                if task_obj.args.input_from == "encoder"
-                else args.decoder_embed_dim
-            )
-            task_decoder = cls.build_multitask_decoder(
-                task_obj.args, task_obj.target_dictionary, in_dim
-            )
-
-            setattr(base_model, f"{task_name}_decoder", task_decoder)
-            decoder_model_cls = (
-                FairseqEncoderModel
-                if task_obj.args.decoder_type == "ctc"
-                else FairseqLanguageModel
-            )
-            base_model.multitask_decoders[task_name] = decoder_model_cls(
-                getattr(base_model, f"{task_name}_decoder")
-            )
-
-        return base_model
-
-    def forward_encoder(self, src_tokens, src_lengths, speaker=None, **kwargs):
-        return self.encoder(
-            src_tokens, src_lengths=src_lengths, tgt_speaker=speaker, **kwargs
-        )
-
-
-@register_model("s2ut_transformer")
-class S2UTTransformerModel(S2STransformerMultitaskModelBase):
-    """
-    Direct speech-to-speech translation model with S2T Transformer encoder + Transformer discrete unit decoder
-    https://arxiv.org/abs/2107.05604
-    """
-
-    @staticmethod
-    def add_args(parser):
-        # input
-        parser.add_argument(
-            "--conv-kernel-sizes",
-            type=str,
-            metavar="N",
-            help="kernel sizes of Conv1d subsampling layers",
-        )
-        parser.add_argument(
-            "--conv-channels",
-            type=int,
-            metavar="N",
-            help="# of channels in Conv1d subsampling layers",
-        )
-        # Transformer
-        parser.add_argument(
-            "--activation-fn",
-            type=str,
-            default="relu",
-            choices=utils.get_available_activation_fns(),
-            help="activation function to use",
-        )
-        parser.add_argument(
-            "--dropout", type=float, metavar="D", help="dropout probability"
-        )
-        parser.add_argument(
-            "--attention-dropout",
-            type=float,
-            metavar="D",
-            help="dropout probability for attention weights",
-        )
-        parser.add_argument(
-            "--activation-dropout",
-            "--relu-dropout",
-            type=float,
-            metavar="D",
-            help="dropout probability after activation in FFN.",
-        )
-        parser.add_argument(
-            "--encoder-embed-dim",
-            type=int,
-            metavar="N",
-            help="encoder embedding dimension",
-        )
-        parser.add_argument(
-            "--encoder-ffn-embed-dim",
-            type=int,
-            metavar="N",
-            help="encoder embedding dimension for FFN",
-        )
-        parser.add_argument(
-            "--encoder-layers", type=int, metavar="N", help="num encoder layers"
-        )
-        parser.add_argument(
-            "--encoder-attention-heads",
-            type=int,
-            metavar="N",
-            help="num encoder attention heads",
-        )
-        parser.add_argument(
-            "--encoder-normalize-before",
-            action="store_true",
-            help="apply layernorm before each encoder block",
-        )
-        parser.add_argument(
-            "--decoder-embed-dim",
-            type=int,
-            metavar="N",
-            help="decoder embedding dimension",
-        )
-        parser.add_argument(
-            "--decoder-ffn-embed-dim",
-            type=int,
-            metavar="N",
-            help="decoder embedding dimension for FFN",
-        )
-        parser.add_argument(
-            "--decoder-layers", type=int, metavar="N", help="num decoder layers"
-        )
-        parser.add_argument(
-            "--decoder-attention-heads",
-            type=int,
-            metavar="N",
-            help="num decoder attention heads",
-        )
-        parser.add_argument(
-            "--decoder-normalize-before",
-            action="store_true",
-            help="apply layernorm before each decoder block",
-        )
-        parser.add_argument(
-            "--share-decoder-input-output-embed",
-            action="store_true",
-            help="share decoder input and output embeddings",
-        )
-        parser.add_argument(
-            "--layernorm-embedding",
-            action="store_true",
-            help="add layernorm to embedding",
-        )
-        parser.add_argument(
-            "--no-scale-embedding",
-            action="store_true",
-            help="if True, dont scale embeddings",
-        )
-        parser.add_argument(
-            "--load-pretrained-encoder-from",
-            type=str,
-            metavar="STR",
-            help="model to take encoder weights from (for initialization)",
-        )
-        parser.add_argument(
-            "--encoder-freezing-updates",
-            type=int,
-            metavar="N",
-            help="freeze encoder for first N updates",
-        )
-        # speaker
-        parser.add_argument(
-            "--speaker-embed-dim",
-            type=int,
-            metavar="N",
-            help="speaker embedding dimension",
-        )
-
-    @classmethod
-    def build_decoder(cls, args, tgt_dict):
-        num_embeddings = len(tgt_dict)
-        padding_idx = tgt_dict.pad()
-        embed_tokens = StackedEmbedding(
-            num_embeddings,
-            args.decoder_embed_dim,
-            padding_idx,
-            num_stacked=args.n_frames_per_step,
-        )
-
-        return TransformerUnitDecoder(
-            args,
-            tgt_dict,
-            embed_tokens,
-        )
-
-    def forward(
-        self,
-        src_tokens,
-        src_lengths,
-        prev_output_tokens,
-        tgt_speaker=None,
-        return_all_hiddens=False,
-    ):
-        encoder_out = self.encoder(
-            src_tokens,
-            src_lengths=src_lengths,
-            tgt_speaker=tgt_speaker,
-            return_all_hiddens=return_all_hiddens,
-        )
-        decoder_out = self.decoder(
-            prev_output_tokens,
-            encoder_out=encoder_out,
-        )
-        if return_all_hiddens:
-            decoder_out[-1]["encoder_states"] = encoder_out["encoder_states"]
-            decoder_out[-1]["encoder_padding_mask"] = encoder_out[
-                "encoder_padding_mask"
-            ]
-        return decoder_out
-
-
-@register_model("s2spect_transformer")
-class S2SpecTTransformerModel(S2STransformerMultitaskModelBase):
-    """
-    Speech-to-spectrogram model with S2T Transformer encoder + TTS Transformer decoder
-    """
-
-    @staticmethod
-    def add_args(parser):
-        # input
-        parser.add_argument(
-            "--conv-kernel-sizes",
-            type=str,
-            metavar="N",
-            help="kernel sizes of Conv1d subsampling layers",
-        )
-        parser.add_argument(
-            "--conv-channels",
-            type=int,
-            metavar="N",
-            help="# of channels in Conv1d subsampling layers",
-        )
-        # Transformer
-        parser.add_argument(
-            "--activation-fn",
-            type=str,
-            default="relu",
-            choices=utils.get_available_activation_fns(),
-            help="activation function to use",
-        )
-        parser.add_argument(
-            "--dropout", type=float, metavar="D", help="dropout probability"
-        )
-        parser.add_argument(
-            "--attention-dropout",
-            type=float,
-            metavar="D",
-            help="dropout probability for attention weights",
-        )
-        parser.add_argument(
-            "--activation-dropout",
-            "--relu-dropout",
-            type=float,
-            metavar="D",
-            help="dropout probability after activation in FFN.",
-        )
-        parser.add_argument(
-            "--encoder-embed-dim",
-            type=int,
-            metavar="N",
-            help="encoder embedding dimension",
-        )
-        parser.add_argument(
-            "--encoder-ffn-embed-dim",
-            type=int,
-            metavar="N",
-            help="encoder embedding dimension for FFN",
-        )
-        parser.add_argument(
-            "--encoder-layers", type=int, metavar="N", help="num encoder layers"
-        )
-        parser.add_argument(
-            "--encoder-attention-heads",
-            type=int,
-            metavar="N",
-            help="num encoder attention heads",
-        )
-        parser.add_argument(
-            "--encoder-normalize-before",
-            action="store_true",
-            help="apply layernorm before each encoder block",
-        )
-        parser.add_argument(
-            "--no-scale-embedding",
-            action="store_true",
-            help="if True, dont scale embeddings",
-        )
-        parser.add_argument(
-            "--load-pretrained-encoder-from",
-            type=str,
-            metavar="STR",
-            help="model to take encoder weights from (for initialization)",
-        )
-        parser.add_argument(
-            "--encoder-freezing-updates",
-            type=int,
-            metavar="N",
-            help="freeze encoder for first N updates",
-        )
-        # speaker
-        parser.add_argument(
-            "--speaker-embed-dim",
-            type=int,
-            metavar="N",
-            help="speaker embedding dimension",
-        )
-        # decoder
-        parser.add_argument("--output-frame-dim", type=int)
-        # decoder prenet
-        parser.add_argument("--prenet-dropout", type=float)
-        parser.add_argument("--prenet-layers", type=int)
-        parser.add_argument("--prenet-dim", type=int)
-        # decoder postnet
-        parser.add_argument("--postnet-dropout", type=float)
-        parser.add_argument("--postnet-layers", type=int)
-        parser.add_argument("--postnet-conv-dim", type=int)
-        parser.add_argument("--postnet-conv-kernel-size", type=int)
-        # decoder transformer layers
-        parser.add_argument("--decoder-transformer-layers", type=int)
-        parser.add_argument("--decoder-embed-dim", type=int)
-        parser.add_argument("--decoder-ffn-embed-dim", type=int)
-        parser.add_argument("--decoder-normalize-before", action="store_true")
-        parser.add_argument("--decoder-attention-heads", type=int)
-
-    @classmethod
-    def build_decoder(cls, args):
-        return TTSTransformerDecoder(args, None, padding_idx=1)
-
-    def forward(
-        self,
-        src_tokens,
-        src_lengths,
-        prev_output_tokens,
-        tgt_speaker=None,
-        incremental_state=None,
-        target_lengths=None,
-        speaker=None,
-        return_all_hiddens=False,
-    ):
-        encoder_out = self.encoder(
-            src_tokens,
-            src_lengths=src_lengths,
-            tgt_speaker=tgt_speaker,
-            return_all_hiddens=return_all_hiddens,
-        )
-        decoder_out = self.decoder(
-            prev_output_tokens,
-            encoder_out=encoder_out,
-            incremental_state=incremental_state,
-            target_lengths=target_lengths,
-            speaker=speaker,
-        )
-        if return_all_hiddens:
-            decoder_out[-1]["encoder_states"] = encoder_out["encoder_states"]
-            decoder_out[-1]["encoder_padding_mask"] = encoder_out[
-                "encoder_padding_mask"
-            ]
-        return decoder_out
-
-
-def base_multitask_text_transformer_decoder_arch(args):
-    args.dropout = getattr(args, "dropout", 0.3)
-    args.decoder_layerdrop = getattr(args, "decoder_layerdrop", 0.0)
-    args.share_decoder_input_output_embed = getattr(
-        args, "share_decoder_input_output_embed", True
-    )
-    args.decoder_embed_dim = getattr(args, "decoder_embed_dim", 256)
-    args.decoder_output_dim = getattr(
-        args, "decoder_output_dim", args.decoder_embed_dim
-    )
-    args.decoder_input_dim = getattr(args, "decoder_input_dim", args.decoder_embed_dim)
-
-    args.max_target_positions = getattr(args, "max_target_positions", 1024)
-    args.no_scale_embedding = getattr(args, "no_scale_embedding", False)
-
-    args.adaptive_input = getattr(args, "adaptive_input", False)
-    args.quant_noise_pq = getattr(args, "quant_noise_pq", 0)
-
-    args.decoder_learned_pos = getattr(args, "decoder_learned_pos", False)
-    args.no_token_positional_embeddings = getattr(
-        args, "no_token_positional_embeddings", False
-    )
-
-    args.decoder_layers = getattr(args, "decoder_layers", 2)
-
-    args.adaptive_softmax_cutoff = getattr(args, "adaptive_softmax_cutoff", None)
-
-    # decoder layer
-    args.activation_dropout = getattr(args, "activation_dropout", args.dropout)
-    args.activation_fn = getattr(args, "activation_fn", "relu")
-    args.decoder_normalize_before = getattr(args, "decoder_normalize_before", True)
-    args.decoder_ffn_embed_dim = getattr(args, "decoder_ffn_embed_dim", 2048)
-
-    args.attention_dropout = getattr(args, "attention_dropout", args.dropout)
-    args.decoder_attention_heads = getattr(args, "decoder_attention_heads", 4)
-
-
-def base_s2st_transformer_encoder_architecture(args):
-    args.encoder_freezing_updates = getattr(args, "encoder_freezing_updates", 0)
-
-    # Convolutional subsampler
-    args.conv_kernel_sizes = getattr(args, "conv_kernel_sizes", "5,5")
-    args.conv_channels = getattr(args, "conv_channels", 1024)
-    # Transformer
-    args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 512)
-    args.encoder_ffn_embed_dim = getattr(args, "encoder_ffn_embed_dim", 2048)
-    args.encoder_layers = getattr(args, "encoder_layers", 12)
-    args.encoder_attention_heads = getattr(args, "encoder_attention_heads", 8)
-    args.encoder_normalize_before = getattr(args, "encoder_normalize_before", True)
-    args.no_scale_embedding = getattr(args, "no_scale_embedding", False)
-
-    args.dropout = getattr(args, "dropout", 0.1)
-    args.attention_dropout = getattr(args, "attention_dropout", args.dropout)
-    args.activation_dropout = getattr(args, "activation_dropout", args.dropout)
-    args.activation_fn = getattr(args, "activation_fn", "relu")
-
-    args.speaker_embed_dim = getattr(args, "speaker_embed_dim", 256)
-
-
-@register_model_architecture(
-    model_name="s2ut_transformer", arch_name="s2ut_transformer"
-)
-def s2ut_architecture_base(args):
-    base_s2st_transformer_encoder_architecture(args)
-
-    # decoder
-    args.decoder_embed_dim = getattr(args, "decoder_embed_dim", args.encoder_embed_dim)
-    args.decoder_ffn_embed_dim = getattr(
-        args, "decoder_ffn_embed_dim", args.encoder_ffn_embed_dim
-    )
-    args.decoder_layers = getattr(args, "decoder_layers", 6)
-    args.decoder_attention_heads = getattr(args, "decoder_attention_heads", 8)
-    args.decoder_normalize_before = getattr(args, "decoder_normalize_before", True)
-    args.decoder_learned_pos = getattr(args, "decoder_learned_pos", False)
-    args.adaptive_softmax_cutoff = getattr(args, "adaptive_softmax_cutoff", None)
-    args.adaptive_softmax_dropout = getattr(args, "adaptive_softmax_dropout", 0)
-    args.share_decoder_input_output_embed = getattr(
-        args, "share_decoder_input_output_embed", False
-    )
-    args.no_token_positional_embeddings = getattr(
-        args, "no_token_positional_embeddings", False
-    )
-    args.adaptive_input = getattr(args, "adaptive_input", False)
-    args.decoder_layerdrop = getattr(args, "decoder_layerdrop", 0.0)
-    args.decoder_output_dim = getattr(
-        args, "decoder_output_dim", args.decoder_embed_dim
-    )
-    args.decoder_input_dim = getattr(args, "decoder_input_dim", args.decoder_embed_dim)
-    args.quant_noise_pq = getattr(args, "quant_noise_pq", 0)
-
-
-@register_model_architecture("s2ut_transformer", "s2ut_transformer_fisher")
-def s2ut_architecture_fisher(args):
-    args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 256)
-    args.encoder_attention_heads = getattr(args, "encoder_attention_heads", 4)
-    args.dropout = getattr(args, "dropout", 0.1)
-
-    s2ut_architecture_base(args)
-
-
-@register_model_architecture(
-    model_name="s2spect_transformer", arch_name="s2spect_transformer"
-)
-def s2spect_architecture_base(args):
-    base_s2st_transformer_encoder_architecture(args)
-
-    # decoder
-    args.output_frame_dim = getattr(args, "output_frame_dim", 80)
-    # decoder prenet
-    args.prenet_dropout = getattr(args, "prenet_dropout", 0.5)
-    args.prenet_layers = getattr(args, "prenet_layers", 2)
-    args.prenet_dim = getattr(args, "prenet_dim", 256)
-    # decoder postnet
-    args.postnet_dropout = getattr(args, "postnet_dropout", 0.5)
-    args.postnet_layers = getattr(args, "postnet_layers", 5)
-    args.postnet_conv_dim = getattr(args, "postnet_conv_dim", 512)
-    args.postnet_conv_kernel_size = getattr(args, "postnet_conv_kernel_size", 5)
-    # decoder transformer layers
-    args.decoder_transformer_layers = getattr(args, "decoder_transformer_layers", 6)
-    args.decoder_embed_dim = getattr(args, "decoder_embed_dim", 512)
-    args.decoder_ffn_embed_dim = getattr(
-        args, "decoder_ffn_embed_dim", 4 * args.decoder_embed_dim
-    )
-    args.decoder_normalize_before = getattr(args, "decoder_normalize_before", False)
-    args.decoder_attention_heads = getattr(args, "decoder_attention_heads", 4)
-
-
-@register_model_architecture("s2spect_transformer", "s2spect_transformer_fisher")
-def s2spect_architecture_fisher(args):
-    args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 256)
-    args.encoder_ffn_embed_dim = getattr(args, "encoder_ffn_embed_dim", 256 * 8)
-    args.encoder_attention_heads = getattr(args, "encoder_attention_heads", 4)
-    args.dropout = getattr(args, "dropout", 0.1)
-
-    # decoder
-    args.prenet_dim = getattr(args, "prenet_dim", 32)
-
-    s2spect_architecture_base(args)

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/speech_to_text/__init__.py

@@ -1,12 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-from .berard import *  # noqa
-from .convtransformer import *  # noqa
-from .multi_modality_model import *  # noqa
-from .s2t_conformer import *  # noqa
-from .s2t_transformer import *  # noqa
-from .s2t_wav_transformer import *  # noqa
-from .xm_transformer import *  # noqa

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/speech_to_text/berard.py

@@ -1,606 +0,0 @@
-#!/usr/bin/env python3
-
-from ast import literal_eval
-from typing import List, Tuple
-
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-from fairseq import checkpoint_utils, utils
-from fairseq.data.data_utils import lengths_to_padding_mask
-from fairseq.models import (
-    FairseqEncoder,
-    FairseqEncoderDecoderModel,
-    FairseqIncrementalDecoder,
-    register_model,
-    register_model_architecture,
-)
-
-
-@register_model("s2t_berard")
-class BerardModel(FairseqEncoderDecoderModel):
-    """Implementation of a model similar to https://arxiv.org/abs/1802.04200
-
-    Paper title: End-to-End Automatic Speech Translation of Audiobooks
-    An implementation is available in tensorflow at
-    https://github.com/eske/seq2seq
-    Relevant files in this implementation are the config
-    (https://github.com/eske/seq2seq/blob/master/config/LibriSpeech/AST.yaml)
-    and the model code
-    (https://github.com/eske/seq2seq/blob/master/translate/models.py).
-    The encoder and decoder try to be close to the original implementation.
-    The attention is an MLP as in Bahdanau et al.
-    (https://arxiv.org/abs/1409.0473).
-    There is no state initialization by averaging the encoder outputs.
-    """
-
-    def __init__(self, encoder, decoder):
-        super().__init__(encoder, decoder)
-
-    @staticmethod
-    def add_args(parser):
-        parser.add_argument(
-            "--input-layers",
-            type=str,
-            metavar="EXPR",
-            help="List of linear layer dimensions. These "
-            "layers are applied to the input features and "
-            "are followed by tanh and possibly dropout.",
-        )
-        parser.add_argument(
-            "--dropout",
-            type=float,
-            metavar="D",
-            help="Dropout probability to use in the encoder/decoder. "
-            "Note that this parameters control dropout in various places, "
-            "there is no fine-grained control for dropout for embeddings "
-            "vs LSTM layers for example.",
-        )
-        parser.add_argument(
-            "--in-channels",
-            type=int,
-            metavar="N",
-            help="Number of encoder input channels. " "Typically value is 1.",
-        )
-        parser.add_argument(
-            "--conv-layers",
-            type=str,
-            metavar="EXPR",
-            help="List of conv layers " "(format: (channels, kernel, stride)).",
-        )
-        parser.add_argument(
-            "--num-blstm-layers",
-            type=int,
-            metavar="N",
-            help="Number of encoder bi-LSTM layers.",
-        )
-        parser.add_argument(
-            "--lstm-size", type=int, metavar="N", help="LSTM hidden size."
-        )
-        parser.add_argument(
-            "--decoder-embed-dim",
-            type=int,
-            metavar="N",
-            help="Embedding dimension of the decoder target tokens.",
-        )
-        parser.add_argument(
-            "--decoder-hidden-dim",
-            type=int,
-            metavar="N",
-            help="Decoder LSTM hidden dimension.",
-        )
-        parser.add_argument(
-            "--decoder-num-layers",
-            type=int,
-            metavar="N",
-            help="Number of decoder LSTM layers.",
-        )
-        parser.add_argument(
-            "--attention-dim",
-            type=int,
-            metavar="N",
-            help="Hidden layer dimension in MLP attention.",
-        )
-        parser.add_argument(
-            "--output-layer-dim",
-            type=int,
-            metavar="N",
-            help="Hidden layer dim for linear layer prior to output projection.",
-        )
-        parser.add_argument(
-            "--load-pretrained-encoder-from",
-            type=str,
-            metavar="STR",
-            help="model to take encoder weights from (for initialization)",
-        )
-        parser.add_argument(
-            "--load-pretrained-decoder-from",
-            type=str,
-            metavar="STR",
-            help="model to take decoder weights from (for initialization)",
-        )
-
-    @classmethod
-    def build_encoder(cls, args, task):
-        encoder = BerardEncoder(
-            input_layers=literal_eval(args.input_layers),
-            conv_layers=literal_eval(args.conv_layers),
-            in_channels=args.input_channels,
-            input_feat_per_channel=args.input_feat_per_channel,
-            num_blstm_layers=args.num_blstm_layers,
-            lstm_size=args.lstm_size,
-            dropout=args.dropout,
-        )
-        if getattr(args, "load_pretrained_encoder_from", None):
-            encoder = checkpoint_utils.load_pretrained_component_from_model(
-                component=encoder, checkpoint=args.load_pretrained_encoder_from
-            )
-        return encoder
-
-    @classmethod
-    def build_decoder(cls, args, task):
-        decoder = LSTMDecoder(
-            dictionary=task.target_dictionary,
-            embed_dim=args.decoder_embed_dim,
-            num_layers=args.decoder_num_layers,
-            hidden_size=args.decoder_hidden_dim,
-            dropout=args.dropout,
-            encoder_output_dim=2 * args.lstm_size,  # bidirectional
-            attention_dim=args.attention_dim,
-            output_layer_dim=args.output_layer_dim,
-        )
-        if getattr(args, "load_pretrained_decoder_from", None):
-            decoder = checkpoint_utils.load_pretrained_component_from_model(
-                component=decoder, checkpoint=args.load_pretrained_decoder_from
-            )
-        return decoder
-
-    @classmethod
-    def build_model(cls, args, task):
-        """Build a new model instance."""
-        encoder = cls.build_encoder(args, task)
-        decoder = cls.build_decoder(args, task)
-
-        return cls(encoder, decoder)
-
-    def get_normalized_probs(self, net_output, log_probs, sample=None):
-        # net_output['encoder_out'] is a (B, T, D) tensor
-        lprobs = super().get_normalized_probs(net_output, log_probs, sample)
-        # lprobs is a (B, T, D) tensor
-        lprobs.batch_first = True
-        return lprobs
-
-
-class BerardEncoder(FairseqEncoder):
-    def __init__(
-        self,
-        input_layers: List[int],
-        conv_layers: List[Tuple[int]],
-        in_channels: int,
-        input_feat_per_channel: int,
-        num_blstm_layers: int,
-        lstm_size: int,
-        dropout: float,
-    ):
-        """
-        Args:
-            input_layers: list of linear layer dimensions. These layers are
-                applied to the input features and are followed by tanh and
-                possibly dropout.
-            conv_layers: list of conv2d layer configurations. A configuration is
-                a tuple (out_channels, conv_kernel_size, stride).
-            in_channels: number of input channels.
-            input_feat_per_channel: number of input features per channel. These
-                are speech features, typically 40 or 80.
-            num_blstm_layers: number of bidirectional LSTM layers.
-            lstm_size: size of the LSTM hidden (and cell) size.
-            dropout: dropout probability. Dropout can be applied after the
-                linear layers and LSTM layers but not to the convolutional
-                layers.
-        """
-        super().__init__(None)
-
-        self.input_layers = nn.ModuleList()
-        in_features = input_feat_per_channel
-        for out_features in input_layers:
-            if dropout > 0:
-                self.input_layers.append(
-                    nn.Sequential(
-                        nn.Linear(in_features, out_features), nn.Dropout(p=dropout)
-                    )
-                )
-            else:
-                self.input_layers.append(nn.Linear(in_features, out_features))
-            in_features = out_features
-
-        self.in_channels = in_channels
-        self.input_dim = input_feat_per_channel
-        self.conv_kernel_sizes_and_strides = []
-        self.conv_layers = nn.ModuleList()
-        lstm_input_dim = input_layers[-1]
-        for conv_layer in conv_layers:
-            out_channels, conv_kernel_size, conv_stride = conv_layer
-            self.conv_layers.append(
-                nn.Conv2d(
-                    in_channels,
-                    out_channels,
-                    conv_kernel_size,
-                    stride=conv_stride,
-                    padding=conv_kernel_size // 2,
-                )
-            )
-            self.conv_kernel_sizes_and_strides.append((conv_kernel_size, conv_stride))
-            in_channels = out_channels
-            lstm_input_dim //= conv_stride
-
-        lstm_input_dim *= conv_layers[-1][0]
-        self.lstm_size = lstm_size
-        self.num_blstm_layers = num_blstm_layers
-        self.lstm = nn.LSTM(
-            input_size=lstm_input_dim,
-            hidden_size=lstm_size,
-            num_layers=num_blstm_layers,
-            dropout=dropout,
-            bidirectional=True,
-        )
-        self.output_dim = 2 * lstm_size  # bidirectional
-        if dropout > 0:
-            self.dropout = nn.Dropout(p=dropout)
-        else:
-            self.dropout = None
-
-    def forward(self, src_tokens, src_lengths=None, **kwargs):
-        """
-        Args
-            src_tokens: padded tensor (B, T, C * feat)
-            src_lengths: tensor of original lengths of input utterances (B,)
-        """
-        bsz, max_seq_len, _ = src_tokens.size()
-        # (B, C, T, feat)
-        x = (
-            src_tokens.view(bsz, max_seq_len, self.in_channels, self.input_dim)
-            .transpose(1, 2)
-            .contiguous()
-        )
-
-        for input_layer in self.input_layers:
-            x = input_layer(x)
-            x = torch.tanh(x)
-
-        for conv_layer in self.conv_layers:
-            x = conv_layer(x)
-
-        bsz, _, output_seq_len, _ = x.size()
-
-        # (B, C, T, feat) -> (B, T, C, feat) -> (T, B, C, feat) ->
-        # (T, B, C * feat)
-        x = x.transpose(1, 2).transpose(0, 1).contiguous().view(output_seq_len, bsz, -1)
-
-        input_lengths = src_lengths.clone()
-        for k, s in self.conv_kernel_sizes_and_strides:
-            p = k // 2
-            input_lengths = (input_lengths.float() + 2 * p - k) / s + 1
-            input_lengths = input_lengths.floor().long()
-
-        packed_x = nn.utils.rnn.pack_padded_sequence(x, input_lengths)
-
-        h0 = x.new(2 * self.num_blstm_layers, bsz, self.lstm_size).zero_()
-        c0 = x.new(2 * self.num_blstm_layers, bsz, self.lstm_size).zero_()
-        packed_outs, _ = self.lstm(packed_x, (h0, c0))
-
-        # unpack outputs and apply dropout
-        x, output_lengths = nn.utils.rnn.pad_packed_sequence(packed_outs)
-        if self.dropout is not None:
-            x = self.dropout(x)
-
-        encoder_padding_mask = (
-            lengths_to_padding_mask(output_lengths).to(src_tokens.device).t()
-        )
-
-        return {
-            "encoder_out": x,  # (T, B, C)
-            "encoder_padding_mask": encoder_padding_mask,  # (T, B)
-        }
-
-    def reorder_encoder_out(self, encoder_out, new_order):
-        encoder_out["encoder_out"] = encoder_out["encoder_out"].index_select(
-            1, new_order
-        )
-        encoder_out["encoder_padding_mask"] = encoder_out[
-            "encoder_padding_mask"
-        ].index_select(1, new_order)
-        return encoder_out
-
-
-class MLPAttention(nn.Module):
-    """The original attention from Badhanau et al. (2014)
-
-    https://arxiv.org/abs/1409.0473, based on a Multi-Layer Perceptron.
-    The attention score between position i in the encoder and position j in the
-    decoder is: alpha_ij = V_a * tanh(W_ae * enc_i + W_ad * dec_j + b_a)
-    """
-
-    def __init__(self, decoder_hidden_state_dim, context_dim, attention_dim):
-        super().__init__()
-
-        self.context_dim = context_dim
-        self.attention_dim = attention_dim
-        # W_ae and b_a
-        self.encoder_proj = nn.Linear(context_dim, self.attention_dim, bias=True)
-        # W_ad
-        self.decoder_proj = nn.Linear(
-            decoder_hidden_state_dim, self.attention_dim, bias=False
-        )
-        # V_a
-        self.to_scores = nn.Linear(self.attention_dim, 1, bias=False)
-
-    def forward(self, decoder_state, source_hids, encoder_padding_mask):
-        """The expected input dimensions are:
-        decoder_state: bsz x decoder_hidden_state_dim
-        source_hids: src_len x bsz x context_dim
-        encoder_padding_mask: src_len x bsz
-        """
-        src_len, bsz, _ = source_hids.size()
-        # (src_len*bsz) x context_dim (to feed through linear)
-        flat_source_hids = source_hids.view(-1, self.context_dim)
-        # (src_len*bsz) x attention_dim
-        encoder_component = self.encoder_proj(flat_source_hids)
-        # src_len x bsz x attention_dim
-        encoder_component = encoder_component.view(src_len, bsz, self.attention_dim)
-        # 1 x bsz x attention_dim
-        decoder_component = self.decoder_proj(decoder_state).unsqueeze(0)
-        # Sum with broadcasting and apply the non linearity
-        # src_len x bsz x attention_dim
-        hidden_att = torch.tanh(
-            (decoder_component + encoder_component).view(-1, self.attention_dim)
-        )
-        # Project onto the reals to get attentions scores (src_len x bsz)
-        attn_scores = self.to_scores(hidden_att).view(src_len, bsz)
-
-        # Mask + softmax (src_len x bsz)
-        if encoder_padding_mask is not None:
-            attn_scores = (
-                attn_scores.float()
-                .masked_fill_(encoder_padding_mask, float("-inf"))
-                .type_as(attn_scores)
-            )  # FP16 support: cast to float and back
-        # srclen x bsz
-        normalized_masked_attn_scores = F.softmax(attn_scores, dim=0)
-
-        # Sum weighted sources (bsz x context_dim)
-        attn_weighted_context = (
-            source_hids * normalized_masked_attn_scores.unsqueeze(2)
-        ).sum(dim=0)
-
-        return attn_weighted_context, normalized_masked_attn_scores
-
-
-class LSTMDecoder(FairseqIncrementalDecoder):
-    def __init__(
-        self,
-        dictionary,
-        embed_dim,
-        num_layers,
-        hidden_size,
-        dropout,
-        encoder_output_dim,
-        attention_dim,
-        output_layer_dim,
-    ):
-        """
-        Args:
-            dictionary: target text dictionary.
-            embed_dim: embedding dimension for target tokens.
-            num_layers: number of LSTM layers.
-            hidden_size: hidden size for LSTM layers.
-            dropout: dropout probability. Dropout can be applied to the
-                embeddings, the LSTM layers, and the context vector.
-            encoder_output_dim: encoder output dimension (hidden size of
-                encoder LSTM).
-            attention_dim: attention dimension for MLP attention.
-            output_layer_dim: size of the linear layer prior to output
-                projection.
-        """
-        super().__init__(dictionary)
-        self.num_layers = num_layers
-        self.hidden_size = hidden_size
-        num_embeddings = len(dictionary)
-        padding_idx = dictionary.pad()
-        self.embed_tokens = nn.Embedding(num_embeddings, embed_dim, padding_idx)
-        if dropout > 0:
-            self.dropout = nn.Dropout(p=dropout)
-        else:
-            self.dropout = None
-
-        self.layers = nn.ModuleList()
-        for layer_id in range(num_layers):
-            input_size = embed_dim if layer_id == 0 else encoder_output_dim
-            self.layers.append(
-                nn.LSTMCell(input_size=input_size, hidden_size=hidden_size)
-            )
-
-        self.context_dim = encoder_output_dim
-        self.attention = MLPAttention(
-            decoder_hidden_state_dim=hidden_size,
-            context_dim=encoder_output_dim,
-            attention_dim=attention_dim,
-        )
-
-        self.deep_output_layer = nn.Linear(
-            hidden_size + encoder_output_dim + embed_dim, output_layer_dim
-        )
-        self.output_projection = nn.Linear(output_layer_dim, num_embeddings)
-
-    def forward(
-        self, prev_output_tokens, encoder_out=None, incremental_state=None, **kwargs
-    ):
-        encoder_padding_mask = encoder_out["encoder_padding_mask"]
-        encoder_outs = encoder_out["encoder_out"]
-
-        if incremental_state is not None:
-            prev_output_tokens = prev_output_tokens[:, -1:]
-        bsz, seqlen = prev_output_tokens.size()
-
-        srclen = encoder_outs.size(0)
-
-        # embed tokens
-        embeddings = self.embed_tokens(prev_output_tokens)
-        x = embeddings
-        if self.dropout is not None:
-            x = self.dropout(x)
-
-        # B x T x C -> T x B x C
-        x = x.transpose(0, 1)
-
-        # initialize previous states (or get from cache during incremental
-        # generation)
-        cached_state = utils.get_incremental_state(
-            self, incremental_state, "cached_state"
-        )
-        if cached_state is not None:
-            prev_hiddens, prev_cells = cached_state
-        else:
-            prev_hiddens = [encoder_out["encoder_out"].mean(dim=0)] * self.num_layers
-            prev_cells = [x.new_zeros(bsz, self.hidden_size)] * self.num_layers
-
-        attn_scores = x.new_zeros(bsz, srclen)
-        attention_outs = []
-        outs = []
-        for j in range(seqlen):
-            input = x[j, :, :]
-            attention_out = None
-            for i, layer in enumerate(self.layers):
-                # the previous state is one layer below except for the bottom
-                # layer where the previous state is the state emitted by the
-                # top layer
-                hidden, cell = layer(
-                    input,
-                    (
-                        prev_hiddens[(i - 1) % self.num_layers],
-                        prev_cells[(i - 1) % self.num_layers],
-                    ),
-                )
-                if self.dropout is not None:
-                    hidden = self.dropout(hidden)
-                prev_hiddens[i] = hidden
-                prev_cells[i] = cell
-                if attention_out is None:
-                    attention_out, attn_scores = self.attention(
-                        hidden, encoder_outs, encoder_padding_mask
-                    )
-                    if self.dropout is not None:
-                        attention_out = self.dropout(attention_out)
-                    attention_outs.append(attention_out)
-                input = attention_out
-
-            # collect the output of the top layer
-            outs.append(hidden)
-
-        # cache previous states (no-op except during incremental generation)
-        utils.set_incremental_state(
-            self, incremental_state, "cached_state", (prev_hiddens, prev_cells)
-        )
-
-        # collect outputs across time steps
-        x = torch.cat(outs, dim=0).view(seqlen, bsz, self.hidden_size)
-        attention_outs_concat = torch.cat(attention_outs, dim=0).view(
-            seqlen, bsz, self.context_dim
-        )
-
-        # T x B x C -> B x T x C
-        x = x.transpose(0, 1)
-        attention_outs_concat = attention_outs_concat.transpose(0, 1)
-
-        # concat LSTM output, attention output and embedding
-        # before output projection
-        x = torch.cat((x, attention_outs_concat, embeddings), dim=2)
-        x = self.deep_output_layer(x)
-        x = torch.tanh(x)
-        if self.dropout is not None:
-            x = self.dropout(x)
-        # project back to size of vocabulary
-        x = self.output_projection(x)
-
-        # to return the full attn_scores tensor, we need to fix the decoder
-        # to account for subsampling input frames
-        # return x, attn_scores
-        return x, None
-
-    def reorder_incremental_state(self, incremental_state, new_order):
-        super().reorder_incremental_state(incremental_state, new_order)
-        cached_state = utils.get_incremental_state(
-            self, incremental_state, "cached_state"
-        )
-        if cached_state is None:
-            return
-
-        def reorder_state(state):
-            if isinstance(state, list):
-                return [reorder_state(state_i) for state_i in state]
-            return state.index_select(0, new_order)
-
-        new_state = tuple(map(reorder_state, cached_state))
-        utils.set_incremental_state(self, incremental_state, "cached_state", new_state)
-
-
-@register_model_architecture(model_name="s2t_berard", arch_name="s2t_berard")
-def berard(args):
-    """The original version: "End-to-End Automatic Speech Translation of
-    Audiobooks" (https://arxiv.org/abs/1802.04200)
-    """
-    args.input_layers = getattr(args, "input_layers", "[256, 128]")
-    args.conv_layers = getattr(args, "conv_layers", "[(16, 3, 2), (16, 3, 2)]")
-    args.num_blstm_layers = getattr(args, "num_blstm_layers", 3)
-    args.lstm_size = getattr(args, "lstm_size", 256)
-    args.dropout = getattr(args, "dropout", 0.2)
-    args.decoder_embed_dim = getattr(args, "decoder_embed_dim", 128)
-    args.decoder_num_layers = getattr(args, "decoder_num_layers", 2)
-    args.decoder_hidden_dim = getattr(args, "decoder_hidden_dim", 512)
-    args.attention_dim = getattr(args, "attention_dim", 512)
-    args.output_layer_dim = getattr(args, "output_layer_dim", 128)
-    args.load_pretrained_encoder_from = getattr(
-        args, "load_pretrained_encoder_from", None
-    )
-    args.load_pretrained_decoder_from = getattr(
-        args, "load_pretrained_decoder_from", None
-    )
-
-
-@register_model_architecture(model_name="s2t_berard", arch_name="s2t_berard_256_3_3")
-def berard_256_3_3(args):
-    """Used in
-    * "Harnessing Indirect Training Data for End-to-End Automatic Speech
-    Translation: Tricks of the Trade" (https://arxiv.org/abs/1909.06515)
-    * "CoVoST: A Diverse Multilingual Speech-To-Text Translation Corpus"
-    (https://arxiv.org/pdf/2002.01320.pdf)
-    * "Self-Supervised Representations Improve End-to-End Speech Translation"
-    (https://arxiv.org/abs/2006.12124)
-    """
-    args.decoder_num_layers = getattr(args, "decoder_num_layers", 3)
-    berard(args)
-
-
-@register_model_architecture(model_name="s2t_berard", arch_name="s2t_berard_512_3_2")
-def berard_512_3_2(args):
-    args.num_blstm_layers = getattr(args, "num_blstm_layers", 3)
-    args.lstm_size = getattr(args, "lstm_size", 512)
-    args.dropout = getattr(args, "dropout", 0.3)
-    args.decoder_embed_dim = getattr(args, "decoder_embed_dim", 256)
-    args.decoder_num_layers = getattr(args, "decoder_num_layers", 2)
-    args.decoder_hidden_dim = getattr(args, "decoder_hidden_dim", 1024)
-    args.attention_dim = getattr(args, "attention_dim", 512)
-    args.output_layer_dim = getattr(args, "output_layer_dim", 256)
-    berard(args)
-
-
-@register_model_architecture(model_name="s2t_berard", arch_name="s2t_berard_512_5_3")
-def berard_512_5_3(args):
-    args.num_blstm_layers = getattr(args, "num_blstm_layers", 5)
-    args.lstm_size = getattr(args, "lstm_size", 512)
-    args.dropout = getattr(args, "dropout", 0.3)
-    args.decoder_embed_dim = getattr(args, "decoder_embed_dim", 256)
-    args.decoder_num_layers = getattr(args, "decoder_num_layers", 3)
-    args.decoder_hidden_dim = getattr(args, "decoder_hidden_dim", 1024)
-    args.attention_dim = getattr(args, "attention_dim", 512)
-    args.output_layer_dim = getattr(args, "output_layer_dim", 256)
-    berard(args)

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/speech_to_text/convtransformer.py

@@ -1,448 +0,0 @@
-#!/usr/bin/env python3
-
-import logging
-import math
-from typing import Dict, List, Optional, Tuple
-
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-from fairseq import checkpoint_utils, utils
-from fairseq.data.data_utils import lengths_to_padding_mask
-from fairseq.models import (
-    FairseqEncoder,
-    FairseqEncoderDecoderModel,
-    register_model,
-    register_model_architecture,
-)
-from fairseq.models.transformer import Embedding, TransformerDecoder
-from fairseq.modules import LayerNorm, PositionalEmbedding, TransformerEncoderLayer
-from torch import Tensor
-
-logger = logging.getLogger(__name__)
-
-
-@register_model("convtransformer")
-class ConvTransformerModel(FairseqEncoderDecoderModel):
-    """
-    Transformer-based Speech translation model from ESPNet-ST
-    https://arxiv.org/abs/2004.10234
-    """
-
-    def __init__(self, encoder, decoder):
-        super().__init__(encoder, decoder)
-
-    @staticmethod
-    def add_args(parser):
-        """Add model-specific arguments to the parser."""
-        parser.add_argument(
-            "--input-feat-per-channel",
-            type=int,
-            metavar="N",
-            help="encoder input dimension per input channel",
-        )
-        parser.add_argument(
-            "--activation-fn",
-            choices=utils.get_available_activation_fns(),
-            help="activation function to use",
-        )
-        parser.add_argument(
-            "--dropout", type=float, metavar="D", help="dropout probability"
-        )
-        parser.add_argument(
-            "--attention-dropout",
-            type=float,
-            metavar="D",
-            help="dropout probability for attention weights",
-        )
-        parser.add_argument(
-            "--activation-dropout",
-            "--relu-dropout",
-            type=float,
-            metavar="D",
-            help="dropout probability after activation in FFN.",
-        )
-        parser.add_argument(
-            "--encoder-embed-dim",
-            type=int,
-            metavar="N",
-            help="encoder embedding dimension",
-        )
-        parser.add_argument(
-            "--encoder-ffn-embed-dim",
-            type=int,
-            metavar="N",
-            help="encoder embedding dimension for FFN",
-        )
-        parser.add_argument(
-            "--encoder-layers", type=int, metavar="N", help="num encoder layers"
-        )
-        parser.add_argument(
-            "--encoder-attention-heads",
-            type=int,
-            metavar="N",
-            help="num encoder attention heads",
-        )
-        parser.add_argument(
-            "--encoder-normalize-before",
-            action="store_true",
-            help="apply layernorm before each encoder block",
-        )
-        parser.add_argument(
-            "--decoder-embed-dim",
-            type=int,
-            metavar="N",
-            help="decoder embedding dimension",
-        )
-        parser.add_argument(
-            "--decoder-ffn-embed-dim",
-            type=int,
-            metavar="N",
-            help="decoder embedding dimension for FFN",
-        )
-        parser.add_argument(
-            "--decoder-layers", type=int, metavar="N", help="num decoder layers"
-        )
-        parser.add_argument(
-            "--decoder-attention-heads",
-            type=int,
-            metavar="N",
-            help="num decoder attention heads",
-        )
-        parser.add_argument(
-            "--decoder-normalize-before",
-            action="store_true",
-            help="apply layernorm before each decoder block",
-        )
-        parser.add_argument(
-            "--decoder-output-dim",
-            type=int,
-            metavar="N",
-            help="decoder output dimension (extra linear layer if different from decoder embed dim)",
-        )
-        parser.add_argument(
-            "--share-decoder-input-output-embed",
-            action="store_true",
-            help="share decoder input and output embeddings",
-        )
-        parser.add_argument(
-            "--layernorm-embedding",
-            action="store_true",
-            help="add layernorm to embedding",
-        )
-        parser.add_argument(
-            "--no-scale-embedding",
-            action="store_true",
-            help="if True, dont scale embeddings",
-        )
-        parser.add_argument(
-            "--load-pretrained-encoder-from",
-            type=str,
-            metavar="STR",
-            help="model to take encoder weights from (for initialization)",
-        )
-        parser.add_argument(
-            "--load-pretrained-decoder-from",
-            type=str,
-            metavar="STR",
-            help="model to take decoder weights from (for initialization)",
-        )
-        parser.add_argument(
-            "--conv-out-channels",
-            type=int,
-            metavar="INT",
-            help="the number of output channels of conv layer",
-        )
-
-    @classmethod
-    def build_encoder(cls, args):
-        encoder = ConvTransformerEncoder(args)
-        if getattr(args, "load_pretrained_encoder_from", None):
-            encoder = checkpoint_utils.load_pretrained_component_from_model(
-                component=encoder, checkpoint=args.load_pretrained_encoder_from
-            )
-        return encoder
-
-    @classmethod
-    def build_decoder(cls, args, task, embed_tokens):
-        decoder = TransformerDecoderNoExtra(args, task.target_dictionary, embed_tokens)
-        if getattr(args, "load_pretrained_decoder_from", None):
-            decoder = checkpoint_utils.load_pretrained_component_from_model(
-                component=decoder, checkpoint=args.load_pretrained_decoder_from
-            )
-        return decoder
-
-    @classmethod
-    def build_model(cls, args, task):
-        """Build a new model instance."""
-
-        # make sure all arguments are present in older models
-        base_architecture(args)
-
-        def build_embedding(dictionary, embed_dim):
-            num_embeddings = len(dictionary)
-            padding_idx = dictionary.pad()
-            return Embedding(num_embeddings, embed_dim, padding_idx)
-
-        decoder_embed_tokens = build_embedding(
-            task.target_dictionary, args.decoder_embed_dim
-        )
-        encoder = cls.build_encoder(args)
-        decoder = cls.build_decoder(args, task, decoder_embed_tokens)
-        return cls(encoder, decoder)
-
-    @staticmethod
-    @torch.jit.unused
-    def set_batch_first(lprobs):
-        lprobs.batch_first = True
-
-    def get_normalized_probs(
-        self,
-        net_output: Tuple[Tensor, Optional[Dict[str, List[Optional[Tensor]]]]],
-        log_probs: bool,
-        sample: Optional[Dict[str, Tensor]] = None,
-    ):
-        # net_output['encoder_out'] is a (B, T, D) tensor
-        lprobs = self.get_normalized_probs_scriptable(net_output, log_probs, sample)
-        if self.training:
-            self.set_batch_first(lprobs)
-        return lprobs
-
-    def output_layout(self):
-        return "BTD"
-
-    """
-    The forward method inherited from the base class has a **kwargs argument in
-    its input, which is not supported in torchscript. This method overrites the forward
-    method definition without **kwargs.
-    """
-
-    def forward(self, src_tokens, src_lengths, prev_output_tokens):
-        encoder_out = self.encoder(src_tokens=src_tokens, src_lengths=src_lengths)
-        decoder_out = self.decoder(
-            prev_output_tokens=prev_output_tokens, encoder_out=encoder_out
-        )
-        return decoder_out
-
-
-class ConvTransformerEncoder(FairseqEncoder):
-    """Conv + Transformer encoder"""
-
-    def __init__(self, args):
-        """Construct an Encoder object."""
-        super().__init__(None)
-
-        self.dropout = args.dropout
-        self.embed_scale = (
-            1.0 if args.no_scale_embedding else math.sqrt(args.encoder_embed_dim)
-        )
-        self.padding_idx = 1
-        self.in_channels = 1
-        self.input_dim = args.input_feat_per_channel
-        self.conv = torch.nn.Sequential(
-            torch.nn.Conv2d(1, args.conv_out_channels, 3, stride=2, padding=3 // 2),
-            torch.nn.ReLU(),
-            torch.nn.Conv2d(
-                args.conv_out_channels,
-                args.conv_out_channels,
-                3,
-                stride=2,
-                padding=3 // 2,
-            ),
-            torch.nn.ReLU(),
-        )
-        transformer_input_dim = self.infer_conv_output_dim(
-            self.in_channels, self.input_dim, args.conv_out_channels
-        )
-        self.out = torch.nn.Linear(transformer_input_dim, args.encoder_embed_dim)
-        self.embed_positions = PositionalEmbedding(
-            args.max_source_positions,
-            args.encoder_embed_dim,
-            self.padding_idx,
-            learned=False,
-        )
-
-        self.transformer_layers = nn.ModuleList([])
-        self.transformer_layers.extend(
-            [TransformerEncoderLayer(args) for i in range(args.encoder_layers)]
-        )
-        if args.encoder_normalize_before:
-            self.layer_norm = LayerNorm(args.encoder_embed_dim)
-        else:
-            self.layer_norm = None
-
-    def pooling_ratio(self):
-        return 4
-
-    def infer_conv_output_dim(self, in_channels, input_dim, out_channels):
-        sample_seq_len = 200
-        sample_bsz = 10
-        x = torch.randn(sample_bsz, in_channels, sample_seq_len, input_dim)
-        x = torch.nn.Conv2d(1, out_channels, 3, stride=2, padding=3 // 2)(x)
-        x = torch.nn.Conv2d(out_channels, out_channels, 3, stride=2, padding=3 // 2)(x)
-        x = x.transpose(1, 2)
-        mb, seq = x.size()[:2]
-        return x.contiguous().view(mb, seq, -1).size(-1)
-
-    def forward(self, src_tokens, src_lengths):
-        """Encode input sequence.
-        :param torch.Tensor xs: input tensor
-        :param torch.Tensor masks: input mask
-        :return: position embedded tensor and mask
-        :rtype Tuple[torch.Tensor, torch.Tensor]:
-        """
-        bsz, max_seq_len, _ = src_tokens.size()
-        x = (
-            src_tokens.view(bsz, max_seq_len, self.in_channels, self.input_dim)
-            .transpose(1, 2)
-            .contiguous()
-        )
-        x = self.conv(x)
-        bsz, _, output_seq_len, _ = x.size()
-        x = x.transpose(1, 2).transpose(0, 1).contiguous().view(output_seq_len, bsz, -1)
-        x = self.out(x)
-        x = self.embed_scale * x
-
-        subsampling_factor = int(max_seq_len * 1.0 / output_seq_len + 0.5)
-        input_len_0 = (src_lengths.float() / subsampling_factor).ceil().long()
-        input_len_1 = x.size(0) * torch.ones([src_lengths.size(0)]).long().to(
-            input_len_0.device
-        )
-        input_lengths = torch.min(input_len_0, input_len_1)
-
-        encoder_padding_mask = lengths_to_padding_mask(input_lengths)
-
-        positions = self.embed_positions(encoder_padding_mask).transpose(0, 1)
-        x += positions
-        x = F.dropout(x, p=self.dropout, training=self.training)
-
-        for layer in self.transformer_layers:
-            x = layer(x, encoder_padding_mask)
-
-        if not encoder_padding_mask.any():
-            maybe_encoder_padding_mask = None
-        else:
-            maybe_encoder_padding_mask = encoder_padding_mask
-
-        return {
-            "encoder_out": [x],
-            "encoder_padding_mask": [maybe_encoder_padding_mask]
-            if maybe_encoder_padding_mask is not None
-            else [],
-            "encoder_embedding": [],
-            "encoder_states": [],
-            "src_tokens": [],
-            "src_lengths": [],
-        }
-
-    @torch.jit.export
-    def reorder_encoder_out(self, encoder_out: Dict[str, List[Tensor]], new_order):
-        """
-        Reorder encoder output according to *new_order*.
-
-        Args:
-            encoder_out: output from the ``forward()`` method
-            new_order (LongTensor): desired order
-
-        Returns:
-            *encoder_out* rearranged according to *new_order*
-        """
-        new_encoder_out = [encoder_out["encoder_out"][0].index_select(1, new_order)]
-        if len(encoder_out["encoder_padding_mask"]) == 0:
-            new_encoder_padding_mask = []
-        else:
-            new_encoder_padding_mask = [
-                (encoder_out["encoder_padding_mask"][0]).index_select(0, new_order)
-            ]
-        if len(encoder_out["encoder_embedding"]) == 0:
-            new_encoder_embedding = []
-        else:
-            new_encoder_embedding = [
-                (encoder_out["encoder_embedding"][0]).index_select(0, new_order)
-            ]
-        encoder_states = encoder_out["encoder_states"]
-        if len(encoder_states) > 0:
-            for idx, state in enumerate(encoder_states):
-                encoder_states[idx] = state.index_select(1, new_order)
-
-        return {
-            "encoder_out": new_encoder_out,
-            "encoder_padding_mask": new_encoder_padding_mask,
-            "encoder_embedding": new_encoder_embedding,
-            "encoder_states": encoder_states,
-            "src_tokens": [],
-            "src_lengths": [],
-        }
-
-
-class TransformerDecoderNoExtra(TransformerDecoder):
-    def extract_features(
-        self,
-        prev_output_tokens,
-        encoder_out: Optional[Dict[str, List[Tensor]]],
-        incremental_state: Optional[Dict[str, Dict[str, Optional[Tensor]]]] = None,
-        full_context_alignment: bool = False,
-        alignment_layer: Optional[int] = None,
-        alignment_heads: Optional[int] = None,
-    ):
-        # call scriptable method from parent class
-        x, _ = self.extract_features_scriptable(
-            prev_output_tokens,
-            encoder_out,
-            incremental_state,
-            full_context_alignment,
-            alignment_layer,
-            alignment_heads,
-        )
-        return x, None
-
-
-@register_model_architecture(model_name="convtransformer", arch_name="convtransformer")
-def base_architecture(args):
-    args.input_feat_per_channel = getattr(args, "input_feat_per_channel", 80)
-    args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 512)
-    args.encoder_ffn_embed_dim = getattr(args, "encoder_ffn_embed_dim", 2048)
-    args.encoder_layers = getattr(args, "encoder_layers", 6)
-    args.encoder_attention_heads = getattr(args, "encoder_attention_heads", 8)
-    args.encoder_normalize_before = getattr(args, "encoder_normalize_before", False)
-    args.decoder_embed_dim = getattr(args, "decoder_embed_dim", args.encoder_embed_dim)
-    args.decoder_ffn_embed_dim = getattr(
-        args, "decoder_ffn_embed_dim", args.encoder_ffn_embed_dim
-    )
-    args.decoder_layers = getattr(args, "decoder_layers", 6)
-    args.decoder_attention_heads = getattr(args, "decoder_attention_heads", 8)
-    args.decoder_normalize_before = getattr(args, "decoder_normalize_before", False)
-    args.decoder_learned_pos = getattr(args, "decoder_learned_pos", False)
-    args.attention_dropout = getattr(args, "attention_dropout", 0.0)
-    args.activation_dropout = getattr(args, "activation_dropout", 0.0)
-    args.activation_fn = getattr(args, "activation_fn", "relu")
-    args.dropout = getattr(args, "dropout", 0.1)
-    args.adaptive_softmax_cutoff = getattr(args, "adaptive_softmax_cutoff", None)
-    args.adaptive_softmax_dropout = getattr(args, "adaptive_softmax_dropout", 0)
-    args.share_decoder_input_output_embed = getattr(
-        args, "share_decoder_input_output_embed", False
-    )
-    args.no_token_positional_embeddings = getattr(
-        args, "no_token_positional_embeddings", False
-    )
-    args.adaptive_input = getattr(args, "adaptive_input", False)
-    args.decoder_layerdrop = getattr(args, "decoder_layerdrop", 0.0)
-
-    args.decoder_output_dim = getattr(
-        args, "decoder_output_dim", args.decoder_embed_dim
-    )
-    args.decoder_input_dim = getattr(args, "decoder_input_dim", args.decoder_embed_dim)
-    args.no_scale_embedding = getattr(args, "no_scale_embedding", False)
-    args.quant_noise_pq = getattr(args, "quant_noise_pq", 0)
-    args.max_source_positions = getattr(args, "max_source_positions", 3000)
-    args.max_target_positions = getattr(args, "max_target_positions", 1024)
-    args.tie_adaptive_weights = getattr(args, "tie_adaptive_weights", False)
-    args.conv_out_channels = getattr(args, "conv_out_channels", args.encoder_embed_dim)
-
-
-@register_model_architecture("convtransformer", "convtransformer_espnet")
-def convtransformer_espnet(args):
-    args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 256)
-    args.encoder_layers = getattr(args, "encoder_layers", 12)
-    args.encoder_attention_heads = getattr(args, "encoder_attention_heads", 4)
-    args.decoder_attention_heads = getattr(args, "decoder_attention_heads", 4)

audio/safeear/safeear_repo/fairseq_ours/fairseq/models/speech_to_text/hub_interface.py

@@ -1,125 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-import logging
-from argparse import Namespace
-from typing import Optional, Tuple, Union
-
-import fairseq.data.audio.feature_transforms.utterance_cmvn as utt_cmvn
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-from fairseq.data import encoders
-from fairseq.data.audio.audio_utils import convert_waveform as convert_wav
-from fairseq.data.audio.audio_utils import (
-    get_fbank,
-)
-from fairseq.data.audio.audio_utils import get_waveform as get_wav
-from fairseq.data.audio.speech_to_text_dataset import SpeechToTextDataset
-
-logger = logging.getLogger(__name__)
-
-
-class S2THubInterface(nn.Module):
-    def __init__(self, cfg, task, model):
-        super().__init__()
-        self.cfg = cfg
-        self.task = task
-        self.model = model
-        self.model.eval()
-        self.generator = self.task.build_generator([self.model], self.cfg)
-
-    @classmethod
-    def get_model_input(cls, task, audio: Union[str, torch.Tensor]):
-        input_type = task.data_cfg.hub.get("input_type", "fbank80")
-        if input_type == "fbank80_w_utt_cmvn":
-            if isinstance(audio, str):
-                feat = utt_cmvn.UtteranceCMVN()(get_fbank(audio))
-                feat = feat.unsqueeze(0)  # T x D -> 1 x T x D
-            else:
-                import torchaudio.compliance.kaldi as kaldi
-
-                feat = kaldi.fbank(audio, num_mel_bins=80).numpy()  # 1 x T x D
-        elif input_type in {"waveform", "standardized_waveform"}:
-            if isinstance(audio, str):
-                feat, sr = get_wav(audio)  # C x T
-                feat, _ = convert_wav(
-                    feat, sr, to_sample_rate=16_000, to_mono=True
-                )  # C x T -> 1 x T
-            else:
-                feat = audio.numpy()
-        else:
-            raise ValueError(f"Unknown value: input_type = {input_type}")
-
-        src_lengths = torch.Tensor([feat.shape[1]]).long()
-        src_tokens = torch.from_numpy(feat)  # 1 x T (x D)
-        if input_type == "standardized_waveform":
-            with torch.no_grad():
-                src_tokens = F.layer_norm(src_tokens, src_tokens.shape)
-
-        return {
-            "net_input": {
-                "src_tokens": src_tokens,
-                "src_lengths": src_lengths,
-                "prev_output_tokens": None,
-            },
-            "target_lengths": None,
-            "speaker": None,
-        }
-
-    @classmethod
-    def detokenize(cls, task, tokens):
-        text = task.tgt_dict.string(tokens)
-        tkn_cfg = task.data_cfg.bpe_tokenizer
-        tokenizer = encoders.build_bpe(Namespace(**tkn_cfg))
-        return text if tokenizer is None else tokenizer.decode(text)
-
-    @classmethod
-    def get_prefix_token(cls, task, lang):
-        prefix_size = int(task.data_cfg.prepend_tgt_lang_tag)
-        prefix_tokens = None
-        if prefix_size > 0:
-            assert lang is not None
-            lang_tag = SpeechToTextDataset.get_lang_tag_idx(lang, task.tgt_dict)
-            prefix_tokens = torch.Tensor([lang_tag]).long().unsqueeze(0)
-        return prefix_tokens
-
-    @classmethod
-    def get_prediction(
-        cls, task, model, generator, sample, tgt_lang=None, synthesize_speech=False
-    ) -> Union[str, Tuple[str, Tuple[torch.Tensor, int]]]:
-        _tgt_lang = tgt_lang or task.data_cfg.hub.get("tgt_lang", None)
-        prefix = cls.get_prefix_token(task, _tgt_lang)
-        pred_tokens = generator.generate([model], sample, prefix_tokens=prefix)
-        pred = cls.detokenize(task, pred_tokens[0][0]["tokens"])
-
-        if synthesize_speech:
-            pfx = f"{_tgt_lang}_" if task.data_cfg.prepend_tgt_lang_tag else ""
-            tts_model_id = task.data_cfg.hub.get(f"{pfx}tts_model_id", None)
-            if tts_model_id is None:
-                logger.warning("TTS model configuration not found")
-            else:
-                _repo, _id = tts_model_id.split(":")
-                tts_model = torch.hub.load(_repo, _id, verbose=False)
-                pred = (pred, tts_model.predict(pred))
-        return pred
-
-    def predict(
-        self,
-        audio: Union[str, torch.Tensor],
-        tgt_lang: Optional[str] = None,
-        synthesize_speech: bool = False,
-    ) -> Union[str, Tuple[str, Tuple[torch.Tensor, int]]]:
-        # `audio` is either a file path or a 1xT Tensor
-        # return either text or (text, synthetic speech)
-        sample = self.get_model_input(self.task, audio)
-        return self.get_prediction(
-            self.task,
-            self.model,
-            self.generator,
-            sample,
-            tgt_lang=tgt_lang,
-            synthesize_speech=synthesize_speech,
-        )