from transformers import AutoTokenizer, AutoModelForCausalLM, Trainer, TrainingArguments from datasets import load_dataset import torch # Charger TES données AIPROD print("📥 Loading AIPROD script dataset...") dataset = load_dataset("AIPROD-C2/aiprod-script-dataset") # Configuration model_name = "distilgpt2" tokenizer = AutoTokenizer.from_pretrained(model_name) model = AutoModelForCausalLM.from_pretrained(model_name) # Ajouter le token de padding tokenizer.pad_token = tokenizer.eos_token # Préparation des données def tokenize_function(examples): return tokenizer(examples["text"], truncation=True, padding="max_length", max_length=128) print("🔧 Tokenizing AIPROD dataset...") tokenized_datasets = dataset.map(tokenize_function, batched=True) # Entraînement training_args = TrainingArguments( output_dir="./scriptmind-v1-output", num_train_epochs=5, per_device_train_batch_size=4, save_steps=500, save_total_limit=2, logging_dir='./logs', logging_steps=50, ) trainer = Trainer( model=model, args=training_args, train_dataset=tokenized_datasets["train"], ) # LANCEMENT ! print("🚀 Starting AIPROD scriptmind-v1 training...") trainer.train() # Sauvegarde finale trainer.save_model("./scriptmind-v1-final") tokenizer.save_pretrained("./scriptmind-v1-final") print("✅ scriptmind-v1 training completed!")