File size: 3,274 Bytes
037a57f
 
 
 
 
 
b92e9ba
 
 
 
 
 
 
 
 
 
 
 
037a57f
 
 
b92e9ba
 
037a57f
b92e9ba
35290f5
037a57f
b92e9ba
 
 
 
037a57f
 
 
b92e9ba
037a57f
 
 
 
 
b92e9ba
 
 
 
 
 
 
35290f5
b92e9ba
 
 
037a57f
b92e9ba
 
 
 
037a57f
 
 
 
b92e9ba
037a57f
b92e9ba
037a57f
b92e9ba
037a57f
 
b92e9ba
35290f5
037a57f
 
 
 
 
 
b92e9ba
 
 
 
037a57f
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
import torch
from datasets import load_dataset
from transformers import AutoTokenizer, AutoModelForCausalLM, TrainingArguments, Trainer
from peft import LoraConfig, get_peft_model, prepare_model_for_int8_training

def main():
    # ---------------------------------------------------------
    # Paso 1: Indicar d贸nde est谩 tu JSON (local o en Hugging Face)
    # ---------------------------------------------------------
    # Opci贸n A: Local
    dataset_path = {"train": "mis_pdfs.json"}
    # Opci贸n B: Hugging Face (ejemplo: alphasoft1/pdf, con mis_pdfs.json dentro)
    # dataset_path = {"train": "alphasoft1/pdf", "data_files": "mis_pdfs.json"}

    # ---------------------------------------------------------
    # Paso 2: Cargar el modelo base
    # ---------------------------------------------------------
    model_name = "meta-llama-3-8b"  # Ajusta al modelo que tengas
    tokenizer = AutoTokenizer.from_pretrained(model_name)
    model = AutoModelForCausalLM.from_pretrained(
        model_name,
        load_in_8bit=True,  # Para entrenamiento 8 bits (bitsandbytes)
        device_map="auto"
    )

    # Preparar el modelo para entrenamiento en 8 bits
    model = prepare_model_for_int8_training(model)

    # ---------------------------------------------------------
    # Paso 3: Configurar LoRA
    # ---------------------------------------------------------
    lora_config = LoraConfig(
        r=8,
        lora_alpha=32,
        target_modules=["q_proj", "v_proj"],  # T铆picos en Llama
        lora_dropout=0.05,
        bias="none",
        task_type="CAUSAL_LM"
    )
    model = get_peft_model(model, lora_config)

    # ---------------------------------------------------------
    # Paso 4: Cargar y tokenizar el dataset
    # ---------------------------------------------------------
    # Si tu JSON est谩 local:
    dataset = load_dataset("json", data_files=dataset_path["train"], split="train")

    # Funci贸n de tokenizaci贸n
    def tokenize_fn(example):
        return tokenizer(example["text"], truncation=True, max_length=512)

    tokenized_dataset = dataset.map(tokenize_fn, batched=True)

    # ---------------------------------------------------------
    # Paso 5: Configuraci贸n de entrenamiento
    # ---------------------------------------------------------
    training_args = TrainingArguments(
        output_dir="llama3-lora-output",
        per_device_train_batch_size=1,
        gradient_accumulation_steps=4,
        num_train_epochs=1,       # Ajusta seg煤n tu caso
        learning_rate=2e-4,
        fp16=True,                # Half precision
        logging_steps=10,
        evaluation_strategy="no", # Cambia a "steps"/"epoch" si tienes validaci贸n
        save_strategy="epoch"
    )

    # Crear el Trainer
    trainer = Trainer(
        model=model,
        args=training_args,
        train_dataset=tokenized_dataset,
        tokenizer=tokenizer
    )

    # ---------------------------------------------------------
    # Paso 6: Entrenar y guardar
    # ---------------------------------------------------------
    trainer.train()
    model.save_pretrained("llama3-lora-output")
    print("Entrenamiento finalizado. Adaptadores guardados en 'llama3-lora-output'.")

if __name__ == "__main__":
    main()