launch.py

import os
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer

def main():
    # Use the current directory where the model files are located
    model_path = os.path.dirname(os.path.abspath(__file__))
    
    print("Loading model and tokenizer...")
    try:
        tokenizer = AutoTokenizer.from_pretrained(model_path)
        
        # Load the model. trust_remote_code=True is included in case of custom architectures
        model = AutoModelForCausalLM.from_pretrained(
            model_path,
            trust_remote_code=True,
            torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32
        )
        
        device = "cuda" if torch.cuda.is_available() else "cpu"
        model.to(device)
        print(f"Model loaded successfully on {device.upper()}!\n")
    except Exception as e:
        print(f"Error loading model: {e}")
        return

    print("--- Chat Session Started (Type 'exit' or 'quit' to end) ---")
    
    while True:
        try:
            user_input = input("You: ").strip()
            if not user_input:
                continue
            if user_input.lower() in ["exit", "quit"]:
                print("Ending session. Goodbye!")
                break
            
            # Tokenize input
            inputs = tokenizer(user_input, return_tensors="pt").to(device)
            
            # Generate response
            with torch.no_grad():
                outputs = model.generate(
                    **inputs,
                    max_new_tokens=128,
                    do_sample=True,
                    temperature=0.6,
                    top_k=40,
                    top_p=0.9,
                    pad_token_id=tokenizer.eos_token_id if tokenizer.eos_token_id is not None else 0
                )
            
            # Decode only the newly generated tokens
            input_length = inputs.input_ids.shape[1]
            response = tokenizer.decode(outputs[0][input_length:], skip_special_tokens=True)
            
            print(f"Model: {response.strip()}\n")
            
        except KeyboardInterrupt:
            print("\nSession interrupted. Goodbye!")
            break

if __name__ == "__main__":
    main()