LLM-fastAPI

Runtime error

LLM-fastAPI / models /transformer /encode_decode /sublayer_connection.py

add files

f3b11f9 almost 2 years ago

597 Bytes

import torch.nn as nn
from models.transformer.encode_decode.layer_norm import LayerNorm


class SublayerConnection(nn.Module):
    """
    A residual connection followed by a layer norm.
    Note for code simplicity the norm is first as opposed to last.
    """

    def __init__(self, size, dropout):
        super(SublayerConnection, self).__init__()
        self.norm = LayerNorm(size)
        self.dropout = nn.Dropout(dropout)

    def forward(self, x, sublayer):
        "Apply residual connection to any sublayer with the same size."
        return x + self.dropout(sublayer(self.norm(x)))

	import torch.nn as nn
	from models.transformer.encode_decode.layer_norm import LayerNorm


	class SublayerConnection(nn.Module):
	"""
	A residual connection followed by a layer norm.
	Note for code simplicity the norm is first as opposed to last.
	"""

	def __init__(self, size, dropout):
	super(SublayerConnection, self).__init__()
	self.norm = LayerNorm(size)
	self.dropout = nn.Dropout(dropout)

	def forward(self, x, sublayer):
	"Apply residual connection to any sublayer with the same size."
	return x + self.dropout(sublayer(self.norm(x)))