ae / gpu_train.py
ChadlieOak's picture
Upload 22 files
c9e8126 verified
Raw History Blame Contribute Delete
6.65 kB
#!/usr/bin/env python3
"""
GPU-Optimized ZRIA Training Script
"""
import os
import sys
import torch
import time
# We're now in the ae directory
def check_gpu():
"""Check GPU availability and capabilities."""
print("πŸ” Checking GPU availability...")
if torch.cuda.is_available():
gpu_count = torch.cuda.device_count()
print(f"βœ… GPU detected: {gpu_count} device(s)")
for i in range(gpu_count):
gpu_name = torch.cuda.get_device_name(i)
gpu_memory = torch.cuda.get_device_properties(i).total_memory / 1e9
print(f" GPU {i}: {gpu_name} ({gpu_memory:.1f} GB)")
device = torch.device('cuda:0')
print(f"πŸš€ Using GPU: {torch.cuda.get_device_name(0)}")
return device
else:
print("ℹ️ No GPU detected, using CPU")
return torch.device('cpu')
def optimize_for_gpu(device):
"""Optimize training parameters based on GPU capabilities."""
if device.type == 'cuda':
gpu_memory = torch.cuda.get_device_properties(0).total_memory / 1e9
if gpu_memory >= 20: # 20GB+ (RTX 4090, A100)
config = {
'dim': 256,
'max_seq_len': 128,
'batch_size': 64,
'epochs': 150,
'learning_rate': 3e-4,
'weight_decay': 0.01
}
print("⚑ Using high-performance configuration (20GB+ GPU)")
elif gpu_memory >= 12: # 12GB+ (RTX 4070 Ti, RTX 3060)
config = {
'dim': 192,
'max_seq_len': 96,
'batch_size': 32,
'epochs': 150,
'learning_rate': 3e-4,
'weight_decay': 0.01
}
print("⚑ Using medium-performance configuration (12GB+ GPU)")
else: # 8GB (RTX 4060 Ti)
config = {
'dim': 128,
'max_seq_len': 64,
'batch_size': 16,
'epochs': 150,
'learning_rate': 3e-4,
'weight_decay': 0.01
}
print("⚑ Using standard configuration (8GB GPU)")
else:
# CPU configuration
config = {
'dim': 128,
'max_seq_len': 64,
'batch_size': 8,
'epochs': 150,
'learning_rate': 3e-4,
'weight_decay': 0.01
}
print("⚑ Using CPU configuration")
return config
def train_with_gpu(file_path, model_name="gpu_trained_model"):
"""Train the ZRIA model with GPU acceleration."""
# Check GPU
device = check_gpu()
# Optimize configuration
config = optimize_for_gpu(device)
print(f"\nπŸš€ Training ZRIA model on: {file_path}")
print(f"πŸ“Š File size: {os.path.getsize(file_path)} bytes")
print(f"πŸ”§ Device: {device}")
print(f"βš™οΈ Configuration: {config}")
# Import the text processor
from text_processor import HierarchicalTextProcessor
# Initialize processor
processor = HierarchicalTextProcessor()
# Process the text file
print("\nπŸ”„ Processing text into structured knowledge...")
start_time = time.time()
knowledge = processor.process_large_text_file(file_path, chunk_size=5000)
process_time = time.time() - start_time
print(f"βœ… Processing completed in {process_time:.1f} seconds")
# Create training data
print("\nπŸ”„ Creating training data...")
training_data = processor.create_training_data(knowledge)
print(f"βœ… Created {len(training_data)} training examples")
# Train the model with GPU optimization
print(f"\n🧠 Training ZRIA model with GPU acceleration...")
from persistent_model import PersistentZRIAModel
# Create model with GPU-optimized config
model = PersistentZRIAModel(model_name)
model.device = device
# Train with optimized parameters
model.train_and_save(training_data, config)
print(f"\nβœ… Training complete! Model saved as '{model_name}'")
return model
def benchmark_training(file_path):
"""Benchmark CPU vs GPU performance."""
print("🏁 Running training benchmark...")
# CPU training
print("\nπŸ“Š CPU Training Benchmark:")
start_time = time.time()
try:
# Temporarily force CPU
original_cuda_available = torch.cuda.is_available
torch.cuda.is_available = lambda: False
train_with_gpu(file_path, "benchmark_cpu")
cpu_time = time.time() - start_time
print(f"⏱️ CPU training time: {cpu_time:.1f} seconds")
# Restore CUDA
torch.cuda.is_available = original_cuda_available
except Exception as e:
print(f"❌ CPU benchmark failed: {e}")
cpu_time = None
# GPU training
print("\nπŸ“Š GPU Training Benchmark:")
start_time = time.time()
try:
train_with_gpu(file_path, "benchmark_gpu")
gpu_time = time.time() - start_time
print(f"⏱️ GPU training time: {gpu_time:.1f} seconds")
if cpu_time and gpu_time:
speedup = cpu_time / gpu_time
print(f"πŸš€ GPU speedup: {speedup:.1f}x faster")
except Exception as e:
print(f"❌ GPU benchmark failed: {e}")
def main():
"""Main function."""
print("πŸš€ GPU-Optimized ZRIA Training")
print("=" * 50)
# Use the sample file
file_path = "sample_text_file.txt"
if not os.path.exists(file_path):
print(f"❌ File not found: {file_path}")
print("Please make sure the sample_text_file.txt exists.")
return
# Check if user wants to benchmark
benchmark_choice = input("\nWould you like to run a performance benchmark? (y/n): ").strip().lower()
if benchmark_choice == 'y':
benchmark_training(file_path)
else:
# Regular training
try:
model = train_with_gpu(file_path, "my_gpu_trained_model")
print(f"\nπŸŽ‰ Success! Your GPU-accelerated model is ready!")
print(f"You can load it anytime using: load_and_use_model('my_gpu_trained_model')")
except Exception as e:
print(f"❌ Error: {e}")
import traceback
traceback.print_exc()
if __name__ == "__main__":
main()