| ---
|
| license: mit
|
| datasets:
|
| - karpathy/tiny_shakespeare
|
| ---
|
| ShakeGPT |
|
|
| **ShakeGPT** is a lightweight, decoder-only Transformer language model trained on the Tiny Shakespeare dataset. It is designed to capture the stylistic patterns, vocabulary, and structure of Shakespearean English at a character level. |
|
|
| ## Model Description |
| * **Architecture:** Transformer Decoder |
| * **Parameters:** ~0.6M |
| * **Training Data:** Tiny Shakespeare (1.6MB of raw text) |
| * **Tokenization:** Character-level |
| * **Context Window:** 128 characters |
|
|
| ## Technical Specifications |
| | Feature | Value | |
| | :--- | :--- | |
| | `n_embd` (Embedding Dimension) | 128 | |
| | `n_layer` (Transformer Blocks) | 3 | |
| | `n_head` (Attention Heads) | 4 | |
| | `block_size` (Context Length) | 128 | |
| | `dropout` | 0.1 | |
|
|
| --- |
|
|
| ## Inference Script |
|
|
| This script initializes the **ShakeGPT** architecture and loads your saved weights to generate new text. |
|
|
| ```python |
| import torch |
| import torch.nn as nn |
| from torch.nn import functional as F |
| import os |
| |
| # ========================================== |
| # HYPERPARAMETERS (Matched to gpt.py) |
| # ========================================== |
| device = 'cpu' |
| n_embd = 128 |
| n_head = 4 |
| n_layer = 3 |
| block_size = 128 # Fixed mismatch |
| dropout = 0.1 |
| weights_path = 'gpt_weights_best.pth' |
| |
| # Load vocab from same source |
| with open('input.txt', 'r', encoding='utf-8') as f: |
| text = f.read() |
| |
| chars = sorted(list(set(text))) |
| vocab_size = len(chars) |
| itos = { i:ch for i,ch in enumerate(chars) } |
| decode = lambda l: ''.join([itos[i] for i in l]) |
| |
| # ========================================== |
| # MODEL ARCHITECTURE (Must be identical) |
| # ========================================== |
| |
| class Head(nn.Module): |
| def __init__(self, head_size): |
| super().__init__() |
| self.key = nn.Linear(n_embd, head_size, bias=False) |
| self.query = nn.Linear(n_embd, head_size, bias=False) |
| self.value = nn.Linear(n_embd, head_size, bias=False) |
| self.register_buffer('tril', torch.tril(torch.ones(block_size, block_size))) |
| self.dropout = nn.Dropout(dropout) |
| |
| def forward(self, x): |
| B,T,C = x.shape |
| k, q, v = self.key(x), self.query(x), self.value(x) |
| wei = q @ k.transpose(-2,-1) * k.shape[-1]**-0.5 |
| wei = wei.masked_fill(self.tril[:T, :T] == 0, float('-inf')) |
| wei = F.softmax(wei, dim=-1) |
| return self.dropout(wei) @ v |
| |
| class MultiHeadAttention(nn.Module): |
| def __init__(self, num_heads, head_size): |
| super().__init__() |
| self.heads = nn.ModuleList([Head(head_size) for _ in range(num_heads)]) |
| self.proj = nn.Linear(head_size * num_heads, n_embd) |
| self.dropout = nn.Dropout(dropout) |
| |
| def forward(self, x): |
| out = torch.cat([h(x) for h in self.heads], dim=-1) |
| return self.dropout(self.proj(out)) |
| |
| class FeedFoward(nn.Module): |
| def __init__(self, n_embd): |
| super().__init__() |
| self.net = nn.Sequential( |
| nn.Linear(n_embd, 4 * n_embd), # Fixed mismatch (4x) |
| nn.GELU(), # Fixed mismatch (GELU) |
| nn.Linear(4 * n_embd, n_embd), |
| nn.Dropout(dropout), |
| ) |
| def forward(self, x): return self.net(x) |
| |
| class Block(nn.Module): |
| def __init__(self, n_embd, n_head): |
| super().__init__() |
| self.sa = MultiHeadAttention(n_head, n_embd // n_head) |
| self.ffwd = FeedFoward(n_embd) |
| self.ln1, self.ln2 = nn.LayerNorm(n_embd), nn.LayerNorm(n_embd) |
| def forward(self, x): |
| x = x + self.sa(self.ln1(x)) |
| return x + self.ffwd(self.ln2(x)) |
| |
| class GPTLanguageModel(nn.Module): |
| def __init__(self): |
| super().__init__() |
| self.token_embedding_table = nn.Embedding(vocab_size, n_embd) |
| self.position_embedding_table = nn.Embedding(block_size, n_embd) |
| self.blocks = nn.Sequential(*[Block(n_embd, n_head=n_head) for _ in range(n_layer)]) |
| self.ln_f = nn.LayerNorm(n_embd) |
| self.lm_head = nn.Linear(n_embd, vocab_size) |
| |
| def forward(self, idx, targets=None): |
| B, T = idx.shape |
| tok_emb = self.token_embedding_table(idx) |
| pos_emb = self.position_embedding_table(torch.arange(T, device=device)) |
| x = self.blocks(tok_emb + pos_emb) |
| logits = self.lm_head(self.ln_f(x)) |
| return logits, None |
| |
| def generate(self, idx, max_new_tokens): |
| for _ in range(max_new_tokens): |
| idx_cond = idx[:, -block_size:] |
| logits, _ = self(idx_cond) |
| probs = F.softmax(logits[:, -1, :], dim=-1) |
| idx_next = torch.multinomial(probs, num_samples=1) |
| idx = torch.cat((idx, idx_next), dim=1) |
| return idx |
| |
| # ========================================== |
| # EXECUTION |
| # ========================================== |
| model = GPTLanguageModel().to(device) |
| |
| if os.path.exists(weights_path): |
| model.load_state_dict(torch.load(weights_path, map_location=device)) |
| model.eval() |
| print(f"Loaded weights from {weights_path}") |
| else: |
| print("Error: Train the model first.") |
| exit() |
| |
| num_tokens = int(input("Tokens to generate: ") or 100) |
| with torch.no_grad(): |
| context = torch.zeros((1, 1), dtype=torch.long, device=device) |
| print("\n--- GENERATED ---\n" + decode(model.generate(context, max_new_tokens=num_tokens)[0].tolist())) |
| ``` |
