File size: 3,274 Bytes
037a57f b92e9ba 037a57f b92e9ba 037a57f b92e9ba 35290f5 037a57f b92e9ba 037a57f b92e9ba 037a57f b92e9ba 35290f5 b92e9ba 037a57f b92e9ba 037a57f b92e9ba 037a57f b92e9ba 037a57f b92e9ba 037a57f b92e9ba 35290f5 037a57f b92e9ba 037a57f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 | import torch
from datasets import load_dataset
from transformers import AutoTokenizer, AutoModelForCausalLM, TrainingArguments, Trainer
from peft import LoraConfig, get_peft_model, prepare_model_for_int8_training
def main():
# ---------------------------------------------------------
# Paso 1: Indicar d贸nde est谩 tu JSON (local o en Hugging Face)
# ---------------------------------------------------------
# Opci贸n A: Local
dataset_path = {"train": "mis_pdfs.json"}
# Opci贸n B: Hugging Face (ejemplo: alphasoft1/pdf, con mis_pdfs.json dentro)
# dataset_path = {"train": "alphasoft1/pdf", "data_files": "mis_pdfs.json"}
# ---------------------------------------------------------
# Paso 2: Cargar el modelo base
# ---------------------------------------------------------
model_name = "meta-llama-3-8b" # Ajusta al modelo que tengas
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForCausalLM.from_pretrained(
model_name,
load_in_8bit=True, # Para entrenamiento 8 bits (bitsandbytes)
device_map="auto"
)
# Preparar el modelo para entrenamiento en 8 bits
model = prepare_model_for_int8_training(model)
# ---------------------------------------------------------
# Paso 3: Configurar LoRA
# ---------------------------------------------------------
lora_config = LoraConfig(
r=8,
lora_alpha=32,
target_modules=["q_proj", "v_proj"], # T铆picos en Llama
lora_dropout=0.05,
bias="none",
task_type="CAUSAL_LM"
)
model = get_peft_model(model, lora_config)
# ---------------------------------------------------------
# Paso 4: Cargar y tokenizar el dataset
# ---------------------------------------------------------
# Si tu JSON est谩 local:
dataset = load_dataset("json", data_files=dataset_path["train"], split="train")
# Funci贸n de tokenizaci贸n
def tokenize_fn(example):
return tokenizer(example["text"], truncation=True, max_length=512)
tokenized_dataset = dataset.map(tokenize_fn, batched=True)
# ---------------------------------------------------------
# Paso 5: Configuraci贸n de entrenamiento
# ---------------------------------------------------------
training_args = TrainingArguments(
output_dir="llama3-lora-output",
per_device_train_batch_size=1,
gradient_accumulation_steps=4,
num_train_epochs=1, # Ajusta seg煤n tu caso
learning_rate=2e-4,
fp16=True, # Half precision
logging_steps=10,
evaluation_strategy="no", # Cambia a "steps"/"epoch" si tienes validaci贸n
save_strategy="epoch"
)
# Crear el Trainer
trainer = Trainer(
model=model,
args=training_args,
train_dataset=tokenized_dataset,
tokenizer=tokenizer
)
# ---------------------------------------------------------
# Paso 6: Entrenar y guardar
# ---------------------------------------------------------
trainer.train()
model.save_pretrained("llama3-lora-output")
print("Entrenamiento finalizado. Adaptadores guardados en 'llama3-lora-output'.")
if __name__ == "__main__":
main()
|