import torch from datasets import load_dataset from transformers import AutoTokenizer, AutoModelForCausalLM, TrainingArguments, Trainer from peft import LoraConfig, get_peft_model, prepare_model_for_int8_training def main(): # --------------------------------------------------------- # Paso 1: Indicar dónde está tu JSON (local o en Hugging Face) # --------------------------------------------------------- # Opción A: Local dataset_path = {"train": "mis_pdfs.json"} # Opción B: Hugging Face (ejemplo: alphasoft1/pdf, con mis_pdfs.json dentro) # dataset_path = {"train": "alphasoft1/pdf", "data_files": "mis_pdfs.json"} # --------------------------------------------------------- # Paso 2: Cargar el modelo base # --------------------------------------------------------- model_name = "meta-llama-3-8b" # Ajusta al modelo que tengas tokenizer = AutoTokenizer.from_pretrained(model_name) model = AutoModelForCausalLM.from_pretrained( model_name, load_in_8bit=True, # Para entrenamiento 8 bits (bitsandbytes) device_map="auto" ) # Preparar el modelo para entrenamiento en 8 bits model = prepare_model_for_int8_training(model) # --------------------------------------------------------- # Paso 3: Configurar LoRA # --------------------------------------------------------- lora_config = LoraConfig( r=8, lora_alpha=32, target_modules=["q_proj", "v_proj"], # Típicos en Llama lora_dropout=0.05, bias="none", task_type="CAUSAL_LM" ) model = get_peft_model(model, lora_config) # --------------------------------------------------------- # Paso 4: Cargar y tokenizar el dataset # --------------------------------------------------------- # Si tu JSON está local: dataset = load_dataset("json", data_files=dataset_path["train"], split="train") # Función de tokenización def tokenize_fn(example): return tokenizer(example["text"], truncation=True, max_length=512) tokenized_dataset = dataset.map(tokenize_fn, batched=True) # --------------------------------------------------------- # Paso 5: Configuración de entrenamiento # --------------------------------------------------------- training_args = TrainingArguments( output_dir="llama3-lora-output", per_device_train_batch_size=1, gradient_accumulation_steps=4, num_train_epochs=1, # Ajusta según tu caso learning_rate=2e-4, fp16=True, # Half precision logging_steps=10, evaluation_strategy="no", # Cambia a "steps"/"epoch" si tienes validación save_strategy="epoch" ) # Crear el Trainer trainer = Trainer( model=model, args=training_args, train_dataset=tokenized_dataset, tokenizer=tokenizer ) # --------------------------------------------------------- # Paso 6: Entrenar y guardar # --------------------------------------------------------- trainer.train() model.save_pretrained("llama3-lora-output") print("Entrenamiento finalizado. Adaptadores guardados en 'llama3-lora-output'.") if __name__ == "__main__": main()