ComputerAssignments/CA6_systematic_generalization/scripts/run_experiments.sh

#!/bin/bash



set -e


RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'


print_status() {
    echo -e "${BLUE}[INFO]${NC} $1"
}

print_success() {
    echo -e "${GREEN}[SUCCESS]${NC} $1"
}

print_warning() {
    echo -e "${YELLOW}[WARNING]${NC} $1"
}

print_error() {
    echo -e "${RED}[ERROR]${NC} $1"
}


command_exists() {
    command -v "$1" >/dev/null 2>&1
}


check_python() {
    if command_exists python3; then
        PYTHON_VERSION=$(python3 --version 2>&1 | cut -d' ' -f2 | cut -d'.' -f1,2)
        print_status "Found Python $PYTHON_VERSION"
        

        if python3 -c "import sys; exit(0 if sys.version_info >= (3, 8) else 1)"; then
            print_success "Python version is compatible"
            return 0
        else
            print_error "Python version must be >= 3.8"
            return 1
        fi
    else
        print_error "Python3 not found. Please install Python 3.8 or higher."
        return 1
    fi
}


setup_venv() {
    print_status "Setting up virtual environment..."
    
    if [ ! -d "venv" ]; then
        python3 -m venv venv
        print_success "Virtual environment created"
    else
        print_status "Virtual environment already exists"
    fi
    
    source venv/bin/activate
    print_success "Virtual environment activated"
}


install_dependencies() {
    print_status "Installing dependencies..."
    

    pip install --upgrade pip
    

    if [ -f "requirements.txt" ]; then
        pip install -r requirements.txt
        print_success "Dependencies installed from requirements.txt"
    else
        print_warning "requirements.txt not found, installing basic dependencies..."
        pip install torch torchvision torchaudio numpy pandas matplotlib seaborn scikit-learn
    fi
}


create_directories() {
    print_status "Creating directory structure..."
    
    directories=(
        "data"
        "results"
        "results/experiments"
        "results/plots"
        "results/models"
        "logs"
        "configs"
        "scripts"
    )
    
    for dir in "${directories[@]}"; do
        mkdir -p "$dir"
    done
    
    print_success "Directory structure created"
}


run_setup() {
    print_status "Running setup script..."
    
    if [ -f "setup_structure.py" ]; then
        python3 setup_structure.py
        print_success "Setup script completed"
    else
        print_warning "setup_structure.py not found, skipping..."
    fi
}


run_experiments() {
    print_status "Starting systematic generalization experiments..."
    

    if [ -f "src/experiments/experiment_runner.py" ]; then
        python3 -c "
import sys
sys.path.append('.')
from src.experiments.experiment_runner import run_systematic_generalization_experiment


print('Running comprehensive systematic generalization experiment...')
results = run_systematic_generalization_experiment('configs/experiment_config.yaml')
print('Experiment completed successfully!')
print(f'Results saved to: results/experiments/')
"
        print_success "Experiments completed successfully"
    else
        print_error "Experiment runner not found at src/experiments/experiment_runner.py"
        return 1
    fi
}


generate_analysis() {
    print_status "Generating additional analysis..."
    
    python3 -c "
import sys
sys.path.append('.')
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd
import json
import os
from pathlib import Path


results_dir = Path('results/experiments')
if results_dir.exists():
    print('Generating additional analysis plots...')
    

    plt.style.use('seaborn-v0_8')
    

    print('Analysis completed successfully!')
else:
    print('No results found for analysis')
"
    
    print_success "Additional analysis completed"
}


create_summary_report() {
    print_status "Creating summary report..."
    
    cat > results/experiments/SUMMARY.md << EOF


- **Date**: $(date)
- **Python Version**: $(python3 --version)
- **PyTorch Version**: $(python3 -c "import torch; print(torch.__version__)" 2>/dev/null || echo "Not available")


\`\`\`
$(tree -I '__pycache__|*.pyc|venv' 2>/dev/null || find . -type d | head -20)
\`\`\`


- \`results/experiments/\`: Experiment results and metrics
- \`results/plots/\`: Visualization plots
- \`results/models/\`: Trained model checkpoints
- \`logs/\`: Experiment logs


1. Review the generated plots in \`results/plots/\`
2. Check the detailed results in \`results/experiments/\`
3. Analyze the systematic generalization gaps
4. Compare different model architectures


- All experiments use systematic splits to test compositional generalization
- Results include both neural and symbolic approaches
- Comprehensive evaluation metrics are provided
EOF
    
    print_success "Summary report created"
}


cleanup() {
    print_status "Cleaning up temporary files..."
    

    find . -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true
    find . -name "*.pyc" -delete 2>/dev/null || true
    
    print_success "Cleanup completed"
}


main() {
    echo "=========================================="
    echo "Systematic Generalization Experiment Pipeline"
    echo "=========================================="
    echo ""
    

    if ! check_python; then
        exit 1
    fi
    

    setup_venv
    

    install_dependencies
    

    create_directories
    

    run_setup
    

    if run_experiments; then
        print_success "All experiments completed successfully!"
        

        generate_analysis
        

        create_summary_report
        

        cleanup
        
        echo ""
        echo "=========================================="
        echo "Experiment Pipeline Completed Successfully!"
        echo "=========================================="
        echo ""
        echo "Results are available in:"
        echo "  - results/experiments/ (detailed results)"
        echo "  - results/plots/ (visualizations)"
        echo "  - results/models/ (trained models)"
        echo "  - logs/ (experiment logs)"
        echo ""
        echo "Check results/experiments/SUMMARY.md for overview"
        echo ""
        
    else
        print_error "Experiments failed!"
        exit 1
    fi
}


case "${1:-}" in
    "setup")
        print_status "Running setup only..."
        check_python
        setup_venv
        install_dependencies
        create_directories
        run_setup
        print_success "Setup completed"
        ;;
    "run")
        print_status "Running experiments only..."
        source venv/bin/activate 2>/dev/null || print_warning "Virtual environment not found, using system Python"
        run_experiments
        ;;
    "analyze")
        print_status "Running analysis only..."
        source venv/bin/activate 2>/dev/null || print_warning "Virtual environment not found, using system Python"
        generate_analysis
        ;;
    "clean")
        print_status "Cleaning up..."
        cleanup
        ;;
    "help"|"-h"|"--help")
        echo "Usage: $0 [command]"
        echo ""
        echo "Commands:"
        echo "  setup    - Setup environment and dependencies only"
        echo "  run      - Run experiments only (requires setup)"
        echo "  analyze  - Generate additional analysis only"
        echo "  clean    - Clean up temporary files"
        echo "  help     - Show this help message"
        echo ""
        echo "If no command is provided, the full pipeline will run."
        ;;
    "")
        main
        ;;
    *)
        print_error "Unknown command: $1"
        echo "Use '$0 help' for usage information"
        exit 1
        ;;
esac