import gradio as gr import torch import torch.nn as nn import tiktoken # Bundle all necessary class definitions for app.py class LayerNorm(nn.Module): def __init__(self, emb_dim): super().__init__() self.eps = 1e-5 self.scale = nn.Parameter(torch.ones(emb_dim)) self.shift = nn.Parameter(torch.zeros(emb_dim)) def forward(self, x): mean = x.mean(dim=-1, keepdim=True) variance = x.var(dim=-1, keepdim=True, unbiased=False) norm_x = (x - mean) / torch.sqrt(variance + self.eps) return self.scale * norm_x + self.shift class GELU(nn.Module): def __init__(self): super().__init__() def forward(self, x): return 0.5 * x * (1 + torch.tanh(torch.sqrt(torch.tensor(2.0 / torch.pi)) * (x + 0.044715 * torch.pow(x, 3)))) class MLP(nn.Module): def __init__(self, cfg): super().__init__() self.fc_1 = nn.Linear(cfg["emb_dim"], 4 * cfg["emb_dim"]) self.gelu = GELU() self.fc_2 = nn.Linear(4 * cfg["emb_dim"], cfg["emb_dim"]) self.drop = nn.Dropout(cfg["drop_rate"]) def forward(self, x): x = self.fc_1(x) x = self.gelu(x) x = self.fc_2(x) x = self.drop(x) return x class SelfAttention(nn.Module): def __init__(self, cfg): super().__init__() self.n_heads = cfg["n_heads"] self.emb_dim = cfg["emb_dim"] self.head_dim = cfg["emb_dim"] // cfg["n_heads"] self.W_qkv = nn.Linear(cfg["emb_dim"], 3 * cfg["emb_dim"], bias=cfg["qkv_bias"]) self.out_proj = nn.Linear(cfg["emb_dim"], cfg["emb_dim"]) self.drop_attn = nn.Dropout(cfg["drop_rate"]) self.drop_resid = nn.Dropout(cfg["drop_rate"]) def forward(self, x): batch_size, seq_len, emb_dim = x.shape qkv = self.W_qkv(x) q, k, v = qkv.chunk(3, dim=-1) q = q.view(batch_size, seq_len, self.n_heads, self.head_dim).transpose(1, 2) k = k.view(batch_size, seq_len, self.n_heads, self.head_dim).transpose(1, 2) v = v.view(batch_size, seq_len, self.n_heads, self.head_dim).transpose(1, 2) attn_scores = torch.matmul(q, k.transpose(-2, -1)) / (self.head_dim**0.5) mask = torch.triu(torch.ones(seq_len, seq_len, device=x.device), diagonal=1).bool() attn_scores.masked_fill_(mask, float('-inf')) attn_weights = torch.softmax(attn_scores, dim=-1) attn_weights = self.drop_attn(attn_weights) context_vec = torch.matmul(attn_weights, v) context_vec = context_vec.transpose(1, 2).contiguous().view(batch_size, seq_len, emb_dim) output = self.out_proj(context_vec) output = self.drop_resid(output) return output class TransformerBlock(nn.Module): def __init__(self, cfg): super().__init__() self.att = SelfAttention(cfg) self.ffn = MLP(cfg) self.norm1 = LayerNorm(cfg["emb_dim"]) self.norm2 = LayerNorm(cfg["emb_dim"]) self.drop = nn.Dropout(cfg["drop_rate"]) def forward(self, x): shortcut = x x = self.norm1(x) x = self.att(x) x = self.drop(x) x = x + shortcut # Add residual connection shortcut = x x = self.norm2(x) x = self.ffn(x) x = self.drop(x) x = x + shortcut # Add residual connection return x class GPTModel(nn.Module): def __init__(self, cfg): super().__init__() self.tok_emb = nn.Embedding(cfg["vocab_size"], cfg["emb_dim"]) self.pos_emb = nn.Embedding(cfg["context_length"], cfg["emb_dim"]) self.drop_emb = nn.Dropout(cfg["drop_rate"]) self.trf_blocks = nn.Sequential( *[TransformerBlock(cfg) for _ in range(cfg["n_layers"])]) self.final_norm = LayerNorm(cfg["emb_dim"]) self.out_head = nn.Linear( cfg["emb_dim"], cfg["vocab_size"], bias=False ) def forward(self, in_idx): batch_size, seq_len = in_idx.shape tok_embeds = self.tok_emb(in_idx) pos_embeds = self.pos_emb(torch.arange(seq_len, device=in_idx.device)) x = tok_embeds + pos_embeds # Shape [batch_size, num_tokens, emb_size] x = self.drop_emb(x) x = self.trf_blocks(x) x = self.final_norm(x) logits = self.out_head(x) return logits def text_to_token_ids(text, tokenizer): encoded_text = tokenizer.encode(text) token_ids = torch.tensor(encoded_text, dtype=torch.long) return token_ids def token_ids_to_text(token_ids, tokenizer): flat_tokens = token_ids.flatten().tolist() decoded_text = tokenizer.decode(flat_tokens) return decoded_text def generate(model, idx, max_new_tokens, context_size, temperature=0.0, top_k=None, eos_id=None): # For-loop is the same as before: Get logits, and only focus on last time step for _ in range(max_new_tokens): idx_cond = idx[:, -context_size:] with torch.no_grad(): logits = model(idx_cond) logits = logits[:, -1, :] # New: Filter logits with top_k sampling if top_k is not None: # Keep only top_k values top_logits, _ = torch.topk(logits, top_k) min_val = top_logits[:, -1] logits = torch.where(logits < min_val, torch.tensor(float("-inf")).to(logits.device), logits) # New: Apply temperature scaling if temperature > 0.0: logits = logits / temperature # Apply softmax to get probabilities probs = torch.softmax(logits, dim=-1) # (batch_size, context_len) # Sample from the distribution idx_next = torch.multinomial(probs, num_samples=1) # (batch_size, 1) # Otherwise same as before: get idx of the vocab entry with the highest logits value else: idx_next = torch.argmax(logits, dim=-1, keepdim=True) # (batch_size, 1) if idx_next == eos_id: # Stop generating early if end-of-sequence token is encountered and eos_id is specified break # Same as before: append sampled index to the running sequence idx = torch.cat((idx, idx_next), dim=1) # (batch_size, num_tokens+1) return idx # Reload the model configuration NEW_CONFIG_SPACE = { "vocab_size": 50257, "context_length": 1024, "emb_dim": 768, "n_heads": 12, "n_layers": 12, "drop_rate": 0.1, "qkv_bias": True # Set to True based on your training configuration } # Instantiate the model architecture nhyapu_gen_space = GPTModel(NEW_CONFIG_SPACE) # Load the state dictionary from the saved file try: # Modified to map model to CPU nhyapu_gen_space.load_state_dict(torch.load("NHYAPU-Gen.pth", map_location=torch.device('cpu'))) nhyapu_gen_space.eval() print("Model loaded successfully!") except FileNotFoundError: print("Error: Model file 'NHYAPU-Gen.pth' not found. Make sure it's in the same directory as your app.py on Hugging Face Spaces.") # For local testing if the file might be in a different path except Exception as e: print(f"An error occurred while loading the model: {e}") tokenizer_space = tiktoken.get_encoding("gpt2") # Define the generation function for Gradio def generate_text(prompt, max_tokens=100, temperature=0.7, top_k=50): device = torch.device("cpu") # Typically run on CPU in Spaces unless a GPU is explicitly selected nhyapu_gen_space.to(device) # Convert prompt to token IDs idx = text_to_token_ids(prompt, tokenizer_space).unsqueeze(0).to(device) # Generate new tokens generated_token_ids = generate( model=nhyapu_gen_space, idx=idx, max_new_tokens=max_tokens, context_size=NEW_CONFIG_SPACE["context_length"], temperature=temperature, top_k=top_k ) # Convert token IDs back to text output_text = token_ids_to_text(generated_token_ids, tokenizer_space) return output_text # Create the Gradio interface iface = gr.Interface( fn=generate_text, inputs=[ gr.Textbox(lines=2, placeholder="Enter your prompt here...", label="Prompt"), gr.Slider(minimum=10, maximum=500, value=100, label="Max New Tokens"), gr.Slider(minimum=0.1, maximum=2.0, value=0.7, label="Temperature"), gr.Slider(minimum=1, maximum=100, value=50, step=1, label="Top K") ], outputs=gr.Textbox(label="Generated Text"), title="NHYAPU-Gen Text Generator", description="Generate text using NHYAPU-Gen model. ." ) # To run this locally, uncomment the line below: # iface.launch()