rodrigova91 commited on
Commit
037a57f
路
verified 路
1 Parent(s): b47eb52

add train script

Browse files
Files changed (1) hide show
  1. train.py +71 -0
train.py ADDED
@@ -0,0 +1,71 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from datasets import load_dataset
3
+ from transformers import AutoTokenizer, AutoModelForCausalLM, TrainingArguments, Trainer
4
+ from peft import LoraConfig, get_peft_model, prepare_model_for_int8_training
5
+
6
+ def main():
7
+ # Nombre del modelo hipot茅tico (aseg煤rate de que exista o de tenerlo local)
8
+ model_name = "meta-llama-3-8b"
9
+
10
+ # 1. Cargar el tokenizer y el modelo base
11
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
12
+ model = AutoModelForCausalLM.from_pretrained(
13
+ model_name,
14
+ load_in_8bit=True, # para entrenar en 8 bits (requiere bitsandbytes)
15
+ device_map="auto" # distribuye autom谩ticamente en la GPU
16
+ )
17
+
18
+ # 2. Preparar el modelo para entrenamiento en 8 bits
19
+ model = prepare_model_for_int8_training(model)
20
+
21
+ # 3. Configurar LoRA
22
+ lora_config = LoraConfig(
23
+ r=8,
24
+ lora_alpha=32,
25
+ target_modules=["q_proj","v_proj"], # M贸dulos t铆picos en la familia Llama
26
+ lora_dropout=0.05,
27
+ bias="none",
28
+ task_type="CAUSAL_LM"
29
+ )
30
+ model = get_peft_model(model, lora_config)
31
+
32
+ # 4. Cargar un dataset (ejemplo JSON local o uno en Hugging Face)
33
+ # Ajusta a tu dataset real
34
+ dataset = load_dataset("json", data_files="mis_pdfs.json", split="train")
35
+
36
+ # 5. Funci贸n de tokenizaci贸n
37
+ def tokenize_fn(examples):
38
+ return tokenizer(examples["text"], truncation=True, max_length=512)
39
+
40
+ tokenized_dataset = dataset.map(tokenize_fn, batched=True)
41
+
42
+ # 6. Configurar par谩metros de entrenamiento
43
+ training_args = TrainingArguments(
44
+ output_dir="llama3-lora-output",
45
+ per_device_train_batch_size=1,
46
+ gradient_accumulation_steps=4,
47
+ num_train_epochs=1, # Ajusta seg煤n tu caso
48
+ learning_rate=2e-4,
49
+ fp16=True, # Usa half precision
50
+ logging_steps=10,
51
+ evaluation_strategy="no", # Cambia a "steps" o "epoch" si tienes validaci贸n
52
+ save_strategy="epoch"
53
+ )
54
+
55
+ # 7. Crear el Trainer
56
+ trainer = Trainer(
57
+ model=model,
58
+ args=training_args,
59
+ train_dataset=tokenized_dataset,
60
+ tokenizer=tokenizer
61
+ )
62
+
63
+ # 8. Entrenar
64
+ trainer.train()
65
+
66
+ # 9. Guardar adaptadores LoRA
67
+ model.save_pretrained("llama3-lora-output")
68
+ print("Entrenamiento finalizado. Adaptadores guardados en 'llama3-lora-output'.")
69
+
70
+ if __name__ == "__main__":
71
+ main()