Delete train_enwik8.py

train_enwik8.py

@@ -1,114 +0,0 @@
-import os
-import time
-import math
-import torch
-from torch.nn import functional as F
-from model_modified import GPTModified
-import numpy as np
-from contextlib import nullcontext
-
-# Import configurations
-from config.char_config import config
-
-def load_data(split):
-    """Load binary data from split file."""
-    filename = os.path.join('data', f'{split}.bin')
-    with open(filename, 'rb') as f:
-        data = np.fromfile(f, dtype=np.uint8)
-    return data
-
-def get_batch(data, block_size, batch_size, device):
-    """Generate a small batch of data of inputs x and targets y."""
-    ix = torch.randint(len(data) - block_size, (batch_size,))
-    x = torch.stack([torch.from_numpy(data[i:i+block_size].astype(np.int64)) for i in ix])
-    y = torch.stack([torch.from_numpy(data[i+1:i+1+block_size].astype(np.int64)) for i in ix])
-    x, y = x.to(device), y.to(device)
-    return x, y
-
-def estimate_loss(model, data, eval_iters, block_size, batch_size, device):
-    """Estimate loss on data split."""
-    out = {}
-    model.eval()
-    losses = torch.zeros(eval_iters)
-    for k in range(eval_iters):
-        X, Y = get_batch(data, block_size, batch_size, device)
-        with torch.no_grad():
-            logits, loss = model(X, Y)
-            losses[k] = loss.item()
-    out = losses.mean()
-    model.train()
-    return out
-
-def convert_to_bpc(loss):
-    """Convert from natural log (nats) to bits per character (bpc)."""
-    return loss / math.log(2)
-
-def main():
-    # System setup
-    device = 'cuda' if torch.cuda.is_available() else 'cpu'
-    print(f"Using device: {device}")
-    
-    dtype = {'float32': torch.float32, 'bfloat16': torch.bfloat16, 'float16': torch.float16}[config['dtype']]
-    ptdtype = {'float32': torch.float32, 'bfloat16': torch.bfloat16, 'float16': torch.float16}[config['dtype']]
-    ctx = nullcontext() if device == 'cpu' else torch.amp.autocast(device_type=device, dtype=ptdtype)
-
-    # Data loading
-    print("Loading data...")
-    train_data = load_data('train')
-    val_data = load_data('val')
-    
-    # Model init
-    print("Initializing model...")
-    model = GPTModified(config)
-    model.to(device)
-    
-    # Optimizer
-    optimizer = torch.optim.AdamW(
-        model.parameters(),
-        lr=config['learning_rate'],
-        betas=(config['beta1'], config['beta2']),
-        weight_decay=config['weight_decay']
-    )
-
-    # Training loop
-    best_val_loss = float('inf')
-    batch_size = 32
-
-    for iter in range(config['max_iters']):
-        # Sample a batch of data
-        xb, yb = get_batch(train_data, config['block_size'], batch_size, device)
-
-        # Forward pass
-        with ctx:
-            logits, loss = model(xb, yb)
-            loss_bpc = convert_to_bpc(loss.item())
-
-        # Backward pass
-        optimizer.zero_grad(set_to_none=True)
-        loss.backward()
-        torch.nn.utils.clip_grad_norm_(model.parameters(), config['grad_clip'])
-        optimizer.step()
-
-        # Logging
-        if iter % config['log_interval'] == 0:
-            print(f"iter {iter}: train loss {loss_bpc:.4f} bpc")
-
-        # Evaluation
-        if iter % config['eval_interval'] == 0:
-            val_loss = estimate_loss(model, val_data, config['eval_iters'], 
-                                   config['block_size'], batch_size, device)
-            val_bpc = convert_to_bpc(val_loss)
-            print(f"iter {iter}: val loss {val_bpc:.4f} bpc")
-            
-            # Save best model
-            if val_bpc < best_val_loss:
-                best_val_loss = val_bpc
-                torch.save({
-                    'model_state_dict': model.state_dict(),
-                    'optimizer_state_dict': optimizer.state_dict(),
-                    'iter': iter,
-                    'best_val_loss': best_val_loss,
-                }, 'best_model.pt')
-
-if __name__ == '__main__':
-    main()