helloworld / app.py
Sanshre's picture
Update app.py
6a62d8b verified
Raw History Blame Contribute Delete
8.8 kB
import gradio as gr
import torch
import torch.nn as nn
import tiktoken
# Bundle all necessary class definitions for app.py
class LayerNorm(nn.Module):
def __init__(self, emb_dim):
super().__init__()
self.eps = 1e-5
self.scale = nn.Parameter(torch.ones(emb_dim))
self.shift = nn.Parameter(torch.zeros(emb_dim))
def forward(self, x):
mean = x.mean(dim=-1, keepdim=True)
variance = x.var(dim=-1, keepdim=True, unbiased=False)
norm_x = (x - mean) / torch.sqrt(variance + self.eps)
return self.scale * norm_x + self.shift
class GELU(nn.Module):
def __init__(self):
super().__init__()
def forward(self, x):
return 0.5 * x * (1 + torch.tanh(torch.sqrt(torch.tensor(2.0 / torch.pi)) * (x + 0.044715 * torch.pow(x, 3))))
class MLP(nn.Module):
def __init__(self, cfg):
super().__init__()
self.fc_1 = nn.Linear(cfg["emb_dim"], 4 * cfg["emb_dim"])
self.gelu = GELU()
self.fc_2 = nn.Linear(4 * cfg["emb_dim"], cfg["emb_dim"])
self.drop = nn.Dropout(cfg["drop_rate"])
def forward(self, x):
x = self.fc_1(x)
x = self.gelu(x)
x = self.fc_2(x)
x = self.drop(x)
return x
class SelfAttention(nn.Module):
def __init__(self, cfg):
super().__init__()
self.n_heads = cfg["n_heads"]
self.emb_dim = cfg["emb_dim"]
self.head_dim = cfg["emb_dim"] // cfg["n_heads"]
self.W_qkv = nn.Linear(cfg["emb_dim"], 3 * cfg["emb_dim"], bias=cfg["qkv_bias"])
self.out_proj = nn.Linear(cfg["emb_dim"], cfg["emb_dim"])
self.drop_attn = nn.Dropout(cfg["drop_rate"])
self.drop_resid = nn.Dropout(cfg["drop_rate"])
def forward(self, x):
batch_size, seq_len, emb_dim = x.shape
qkv = self.W_qkv(x)
q, k, v = qkv.chunk(3, dim=-1)
q = q.view(batch_size, seq_len, self.n_heads, self.head_dim).transpose(1, 2)
k = k.view(batch_size, seq_len, self.n_heads, self.head_dim).transpose(1, 2)
v = v.view(batch_size, seq_len, self.n_heads, self.head_dim).transpose(1, 2)
attn_scores = torch.matmul(q, k.transpose(-2, -1)) / (self.head_dim**0.5)
mask = torch.triu(torch.ones(seq_len, seq_len, device=x.device), diagonal=1).bool()
attn_scores.masked_fill_(mask, float('-inf'))
attn_weights = torch.softmax(attn_scores, dim=-1)
attn_weights = self.drop_attn(attn_weights)
context_vec = torch.matmul(attn_weights, v)
context_vec = context_vec.transpose(1, 2).contiguous().view(batch_size, seq_len, emb_dim)
output = self.out_proj(context_vec)
output = self.drop_resid(output)
return output
class TransformerBlock(nn.Module):
def __init__(self, cfg):
super().__init__()
self.att = SelfAttention(cfg)
self.ffn = MLP(cfg)
self.norm1 = LayerNorm(cfg["emb_dim"])
self.norm2 = LayerNorm(cfg["emb_dim"])
self.drop = nn.Dropout(cfg["drop_rate"])
def forward(self, x):
shortcut = x
x = self.norm1(x)
x = self.att(x)
x = self.drop(x)
x = x + shortcut # Add residual connection
shortcut = x
x = self.norm2(x)
x = self.ffn(x)
x = self.drop(x)
x = x + shortcut # Add residual connection
return x
class GPTModel(nn.Module):
def __init__(self, cfg):
super().__init__()
self.tok_emb = nn.Embedding(cfg["vocab_size"], cfg["emb_dim"])
self.pos_emb = nn.Embedding(cfg["context_length"], cfg["emb_dim"])
self.drop_emb = nn.Dropout(cfg["drop_rate"])
self.trf_blocks = nn.Sequential(
*[TransformerBlock(cfg) for _ in range(cfg["n_layers"])])
self.final_norm = LayerNorm(cfg["emb_dim"])
self.out_head = nn.Linear(
cfg["emb_dim"], cfg["vocab_size"], bias=False
)
def forward(self, in_idx):
batch_size, seq_len = in_idx.shape
tok_embeds = self.tok_emb(in_idx)
pos_embeds = self.pos_emb(torch.arange(seq_len, device=in_idx.device))
x = tok_embeds + pos_embeds # Shape [batch_size, num_tokens, emb_size]
x = self.drop_emb(x)
x = self.trf_blocks(x)
x = self.final_norm(x)
logits = self.out_head(x)
return logits
def text_to_token_ids(text, tokenizer):
encoded_text = tokenizer.encode(text)
token_ids = torch.tensor(encoded_text, dtype=torch.long)
return token_ids
def token_ids_to_text(token_ids, tokenizer):
flat_tokens = token_ids.flatten().tolist()
decoded_text = tokenizer.decode(flat_tokens)
return decoded_text
def generate(model, idx, max_new_tokens, context_size, temperature=0.0, top_k=None, eos_id=None):
# For-loop is the same as before: Get logits, and only focus on last time step
for _ in range(max_new_tokens):
idx_cond = idx[:, -context_size:]
with torch.no_grad():
logits = model(idx_cond)
logits = logits[:, -1, :]
# New: Filter logits with top_k sampling
if top_k is not None:
# Keep only top_k values
top_logits, _ = torch.topk(logits, top_k)
min_val = top_logits[:, -1]
logits = torch.where(logits < min_val, torch.tensor(float("-inf")).to(logits.device), logits)
# New: Apply temperature scaling
if temperature > 0.0:
logits = logits / temperature
# Apply softmax to get probabilities
probs = torch.softmax(logits, dim=-1) # (batch_size, context_len)
# Sample from the distribution
idx_next = torch.multinomial(probs, num_samples=1) # (batch_size, 1)
# Otherwise same as before: get idx of the vocab entry with the highest logits value
else:
idx_next = torch.argmax(logits, dim=-1, keepdim=True) # (batch_size, 1)
if idx_next == eos_id: # Stop generating early if end-of-sequence token is encountered and eos_id is specified
break
# Same as before: append sampled index to the running sequence
idx = torch.cat((idx, idx_next), dim=1) # (batch_size, num_tokens+1)
return idx
# Reload the model configuration
NEW_CONFIG_SPACE = {
"vocab_size": 50257,
"context_length": 1024,
"emb_dim": 768,
"n_heads": 12,
"n_layers": 12,
"drop_rate": 0.1,
"qkv_bias": True # Set to True based on your training configuration
}
# Instantiate the model architecture
nhyapu_gen_space = GPTModel(NEW_CONFIG_SPACE)
# Load the state dictionary from the saved file
try:
# Modified to map model to CPU
nhyapu_gen_space.load_state_dict(torch.load("NHYAPU-Gen.pth", map_location=torch.device('cpu')))
nhyapu_gen_space.eval()
print("Model loaded successfully!")
except FileNotFoundError:
print("Error: Model file 'NHYAPU-Gen.pth' not found. Make sure it's in the same directory as your app.py on Hugging Face Spaces.")
# For local testing if the file might be in a different path
except Exception as e:
print(f"An error occurred while loading the model: {e}")
tokenizer_space = tiktoken.get_encoding("gpt2")
# Define the generation function for Gradio
def generate_text(prompt, max_tokens=100, temperature=0.7, top_k=50):
device = torch.device("cpu") # Typically run on CPU in Spaces unless a GPU is explicitly selected
nhyapu_gen_space.to(device)
# Convert prompt to token IDs
idx = text_to_token_ids(prompt, tokenizer_space).unsqueeze(0).to(device)
# Generate new tokens
generated_token_ids = generate(
model=nhyapu_gen_space,
idx=idx,
max_new_tokens=max_tokens,
context_size=NEW_CONFIG_SPACE["context_length"],
temperature=temperature,
top_k=top_k
)
# Convert token IDs back to text
output_text = token_ids_to_text(generated_token_ids, tokenizer_space)
return output_text
# Create the Gradio interface
iface = gr.Interface(
fn=generate_text,
inputs=[
gr.Textbox(lines=2, placeholder="Enter your prompt here...", label="Prompt"),
gr.Slider(minimum=10, maximum=500, value=100, label="Max New Tokens"),
gr.Slider(minimum=0.1, maximum=2.0, value=0.7, label="Temperature"),
gr.Slider(minimum=1, maximum=100, value=50, step=1, label="Top K")
],
outputs=gr.Textbox(label="Generated Text"),
title="NHYAPU-Gen Text Generator",
description="Generate text using NHYAPU-Gen model. ."
)
# To run this locally, uncomment the line below:
# iface.launch()