llama3 / train.py
rodrigova91's picture
Update train.py
b92e9ba verified
Raw History Blame Contribute Delete
3.27 kB
import torch
from datasets import load_dataset
from transformers import AutoTokenizer, AutoModelForCausalLM, TrainingArguments, Trainer
from peft import LoraConfig, get_peft_model, prepare_model_for_int8_training
def main():
# ---------------------------------------------------------
# Paso 1: Indicar d贸nde est谩 tu JSON (local o en Hugging Face)
# ---------------------------------------------------------
# Opci贸n A: Local
dataset_path = {"train": "mis_pdfs.json"}
# Opci贸n B: Hugging Face (ejemplo: alphasoft1/pdf, con mis_pdfs.json dentro)
# dataset_path = {"train": "alphasoft1/pdf", "data_files": "mis_pdfs.json"}
# ---------------------------------------------------------
# Paso 2: Cargar el modelo base
# ---------------------------------------------------------
model_name = "meta-llama-3-8b" # Ajusta al modelo que tengas
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForCausalLM.from_pretrained(
model_name,
load_in_8bit=True, # Para entrenamiento 8 bits (bitsandbytes)
device_map="auto"
)
# Preparar el modelo para entrenamiento en 8 bits
model = prepare_model_for_int8_training(model)
# ---------------------------------------------------------
# Paso 3: Configurar LoRA
# ---------------------------------------------------------
lora_config = LoraConfig(
r=8,
lora_alpha=32,
target_modules=["q_proj", "v_proj"], # T铆picos en Llama
lora_dropout=0.05,
bias="none",
task_type="CAUSAL_LM"
)
model = get_peft_model(model, lora_config)
# ---------------------------------------------------------
# Paso 4: Cargar y tokenizar el dataset
# ---------------------------------------------------------
# Si tu JSON est谩 local:
dataset = load_dataset("json", data_files=dataset_path["train"], split="train")
# Funci贸n de tokenizaci贸n
def tokenize_fn(example):
return tokenizer(example["text"], truncation=True, max_length=512)
tokenized_dataset = dataset.map(tokenize_fn, batched=True)
# ---------------------------------------------------------
# Paso 5: Configuraci贸n de entrenamiento
# ---------------------------------------------------------
training_args = TrainingArguments(
output_dir="llama3-lora-output",
per_device_train_batch_size=1,
gradient_accumulation_steps=4,
num_train_epochs=1, # Ajusta seg煤n tu caso
learning_rate=2e-4,
fp16=True, # Half precision
logging_steps=10,
evaluation_strategy="no", # Cambia a "steps"/"epoch" si tienes validaci贸n
save_strategy="epoch"
)
# Crear el Trainer
trainer = Trainer(
model=model,
args=training_args,
train_dataset=tokenized_dataset,
tokenizer=tokenizer
)
# ---------------------------------------------------------
# Paso 6: Entrenar y guardar
# ---------------------------------------------------------
trainer.train()
model.save_pretrained("llama3-lora-output")
print("Entrenamiento finalizado. Adaptadores guardados en 'llama3-lora-output'.")
if __name__ == "__main__":
main()