ys / modeling_ys.py

Publication initiale de Ys et de sa fiche

eefb08d verified about 1 month ago

4.6 kB

# Source : LLMs-from-scratch/ch05/10_llm-training-speed/01_opt_single_gpu.py
# Copie telle quelle : même GPT que le chapitre 4, mais avec les optimisations
# de vitesse du repo :
#   - PyTorchMultiHeadAttention : projection qkv unique + scaled_dot_product_attention
#     (FlashAttention) avec masque causal calculé à la volée
#   - nn.LayerNorm et nn.GELU(approximate="tanh") natifs de PyTorch
# Mêmes hyperparamètres (cfg), mêmes formules, beaucoup plus rapide et compatible bfloat16.

import torch
import torch.nn as nn


#####################################
# Chapter 3
#####################################
class PyTorchMultiHeadAttention(nn.Module):
    def __init__(self, d_in, d_out, num_heads, dropout=0.0, qkv_bias=False):
        super().__init__()

        assert d_out % num_heads == 0, "d_out is indivisible by num_heads"

        self.num_heads = num_heads
        self.head_dim = d_out // num_heads
        self.d_out = d_out

        self.qkv = nn.Linear(d_in, 3 * d_out, bias=qkv_bias)
        self.proj = nn.Linear(d_out, d_out)
        self.dropout = dropout

    def forward(self, x):
        batch_size, num_tokens, embed_dim = x.shape

        # (b, num_tokens, embed_dim) --> (b, num_tokens, 3 * embed_dim)
        qkv = self.qkv(x)

        # (b, num_tokens, 3 * embed_dim) --> (b, num_tokens, 3, num_heads, head_dim)
        qkv = qkv.view(batch_size, num_tokens, 3, self.num_heads, self.head_dim)

        # (b, num_tokens, 3, num_heads, head_dim) --> (3, b, num_heads, num_tokens, head_dim)
        qkv = qkv.permute(2, 0, 3, 1, 4)

        # (3, b, num_heads, num_tokens, head_dim) -> 3 times (b, num_heads, num_tokens, head_dim)
        queries, keys, values = qkv

        use_dropout = 0. if not self.training else self.dropout

        context_vec = nn.functional.scaled_dot_product_attention(
            queries, keys, values, attn_mask=None, dropout_p=use_dropout, is_causal=True)

        # Combine heads, where self.d_out = self.num_heads * self.head_dim
        context_vec = context_vec.transpose(1, 2).contiguous().view(batch_size, num_tokens, self.d_out)

        context_vec = self.proj(context_vec)

        return context_vec


#####################################
# Chapter 4
#####################################


class FeedForward(nn.Module):
    def __init__(self, cfg):
        super().__init__()
        self.layers = nn.Sequential(
            nn.Linear(cfg["emb_dim"], 4 * cfg["emb_dim"]),
            nn.GELU(approximate="tanh"),
            nn.Linear(4 * cfg["emb_dim"], cfg["emb_dim"]),
        )

    def forward(self, x):
        return self.layers(x)


class TransformerBlock(nn.Module):
    def __init__(self, cfg):
        super().__init__()
        self.att = PyTorchMultiHeadAttention(
            d_in=cfg["emb_dim"],
            d_out=cfg["emb_dim"],
            num_heads=cfg["n_heads"],
            dropout=cfg["drop_rate"],
            qkv_bias=cfg["qkv_bias"])
        self.ff = FeedForward(cfg)
        self.norm1 = nn.LayerNorm(cfg["emb_dim"])
        self.norm2 = nn.LayerNorm(cfg["emb_dim"])
        self.drop_shortcut = nn.Dropout(cfg["drop_rate"])

    def forward(self, x):
        # Shortcut connection for attention block
        shortcut = x
        x = self.norm1(x)
        x = self.att(x)   # Shape [batch_size, num_tokens, emb_size]
        x = self.drop_shortcut(x)
        x = x + shortcut  # Add the original input back

        # Shortcut connection for feed-forward block
        shortcut = x
        x = self.norm2(x)
        x = self.ff(x)
        x = self.drop_shortcut(x)
        x = x + shortcut  # Add the original input back

        return x


class GPTModel(nn.Module):
    def __init__(self, cfg):
        super().__init__()
        self.tok_emb = nn.Embedding(cfg["vocab_size"], cfg["emb_dim"])
        self.pos_emb = nn.Embedding(cfg["context_length"], cfg["emb_dim"])
        self.drop_emb = nn.Dropout(cfg["drop_rate"])

        self.trf_blocks = nn.Sequential(
            *[TransformerBlock(cfg) for _ in range(cfg["n_layers"])])

        self.final_norm = nn.LayerNorm(cfg["emb_dim"])
        self.out_head = nn.Linear(cfg["emb_dim"], cfg["vocab_size"], bias=False)

    def forward(self, in_idx):
        batch_size, seq_len = in_idx.shape
        tok_embeds = self.tok_emb(in_idx)
        pos_embeds = self.pos_emb(torch.arange(seq_len, device=in_idx.device))
        x = tok_embeds + pos_embeds  # Shape [batch_size, num_tokens, emb_size]
        x = self.drop_emb(x)
        x = self.trf_blocks(x)
        x = self.final_norm(x)
        logits = self.out_head(x)
        return logits

	# Source : LLMs-from-scratch/ch05/10_llm-training-speed/01_opt_single_gpu.py
	# Copie telle quelle : même GPT que le chapitre 4, mais avec les optimisations
	# de vitesse du repo :
	# - PyTorchMultiHeadAttention : projection qkv unique + scaled_dot_product_attention
	# (FlashAttention) avec masque causal calculé à la volée
	# - nn.LayerNorm et nn.GELU(approximate="tanh") natifs de PyTorch
	# Mêmes hyperparamètres (cfg), mêmes formules, beaucoup plus rapide et compatible bfloat16.

	import torch
	import torch.nn as nn


	#####################################
	# Chapter 3
	#####################################
	class PyTorchMultiHeadAttention(nn.Module):
	def __init__(self, d_in, d_out, num_heads, dropout=0.0, qkv_bias=False):
	super().__init__()

	assert d_out % num_heads == 0, "d_out is indivisible by num_heads"

	self.num_heads = num_heads
	self.head_dim = d_out // num_heads
	self.d_out = d_out

	self.qkv = nn.Linear(d_in, 3 * d_out, bias=qkv_bias)
	self.proj = nn.Linear(d_out, d_out)
	self.dropout = dropout

	def forward(self, x):
	batch_size, num_tokens, embed_dim = x.shape

	# (b, num_tokens, embed_dim) --> (b, num_tokens, 3 * embed_dim)
	qkv = self.qkv(x)

	# (b, num_tokens, 3 * embed_dim) --> (b, num_tokens, 3, num_heads, head_dim)
	qkv = qkv.view(batch_size, num_tokens, 3, self.num_heads, self.head_dim)

	# (b, num_tokens, 3, num_heads, head_dim) --> (3, b, num_heads, num_tokens, head_dim)
	qkv = qkv.permute(2, 0, 3, 1, 4)

	# (3, b, num_heads, num_tokens, head_dim) -> 3 times (b, num_heads, num_tokens, head_dim)
	queries, keys, values = qkv

	use_dropout = 0. if not self.training else self.dropout

	context_vec = nn.functional.scaled_dot_product_attention(
	queries, keys, values, attn_mask=None, dropout_p=use_dropout, is_causal=True)

	# Combine heads, where self.d_out = self.num_heads * self.head_dim
	context_vec = context_vec.transpose(1, 2).contiguous().view(batch_size, num_tokens, self.d_out)

	context_vec = self.proj(context_vec)

	return context_vec


	#####################################
	# Chapter 4
	#####################################


	class FeedForward(nn.Module):
	def __init__(self, cfg):
	super().__init__()
	self.layers = nn.Sequential(
	nn.Linear(cfg["emb_dim"], 4 * cfg["emb_dim"]),
	nn.GELU(approximate="tanh"),
	nn.Linear(4 * cfg["emb_dim"], cfg["emb_dim"]),
	)

	def forward(self, x):
	return self.layers(x)


	class TransformerBlock(nn.Module):
	def __init__(self, cfg):
	super().__init__()
	self.att = PyTorchMultiHeadAttention(
	d_in=cfg["emb_dim"],
	d_out=cfg["emb_dim"],
	num_heads=cfg["n_heads"],
	dropout=cfg["drop_rate"],
	qkv_bias=cfg["qkv_bias"])
	self.ff = FeedForward(cfg)
	self.norm1 = nn.LayerNorm(cfg["emb_dim"])
	self.norm2 = nn.LayerNorm(cfg["emb_dim"])
	self.drop_shortcut = nn.Dropout(cfg["drop_rate"])

	def forward(self, x):
	# Shortcut connection for attention block
	shortcut = x
	x = self.norm1(x)
	x = self.att(x) # Shape [batch_size, num_tokens, emb_size]
	x = self.drop_shortcut(x)
	x = x + shortcut # Add the original input back

	# Shortcut connection for feed-forward block
	shortcut = x
	x = self.norm2(x)
	x = self.ff(x)
	x = self.drop_shortcut(x)
	x = x + shortcut # Add the original input back

	return x


	class GPTModel(nn.Module):
	def __init__(self, cfg):
	super().__init__()
	self.tok_emb = nn.Embedding(cfg["vocab_size"], cfg["emb_dim"])
	self.pos_emb = nn.Embedding(cfg["context_length"], cfg["emb_dim"])
	self.drop_emb = nn.Dropout(cfg["drop_rate"])

	self.trf_blocks = nn.Sequential(
	*[TransformerBlock(cfg) for _ in range(cfg["n_layers"])])

	self.final_norm = nn.LayerNorm(cfg["emb_dim"])
	self.out_head = nn.Linear(cfg["emb_dim"], cfg["vocab_size"], bias=False)

	def forward(self, in_idx):
	batch_size, seq_len = in_idx.shape
	tok_embeds = self.tok_emb(in_idx)
	pos_embeds = self.pos_emb(torch.arange(seq_len, device=in_idx.device))
	x = tok_embeds + pos_embeds # Shape [batch_size, num_tokens, emb_size]
	x = self.drop_emb(x)
	x = self.trf_blocks(x)
	x = self.final_norm(x)
	logits = self.out_head(x)
	return logits