Download train.py from alphasoft91/llama3: direct link, hf CLI and curl.
- Browser
- Download file 3.27 kB
-
https://huggingface.co/alphasoft91/llama3/resolve/main/train.py
- Command line
-
hf download hf://alphasoft91/llama3/train.py
-
curl -L -o train.py https://huggingface.co/alphasoft91/llama3/resolve/main/train.py
3.27 kB
| import torch | |
| from datasets import load_dataset | |
| from transformers import AutoTokenizer, AutoModelForCausalLM, TrainingArguments, Trainer | |
| from peft import LoraConfig, get_peft_model, prepare_model_for_int8_training | |
| def main(): | |
| # --------------------------------------------------------- | |
| # Paso 1: Indicar d贸nde est谩 tu JSON (local o en Hugging Face) | |
| # --------------------------------------------------------- | |
| # Opci贸n A: Local | |
| dataset_path = {"train": "mis_pdfs.json"} | |
| # Opci贸n B: Hugging Face (ejemplo: alphasoft1/pdf, con mis_pdfs.json dentro) | |
| # dataset_path = {"train": "alphasoft1/pdf", "data_files": "mis_pdfs.json"} | |
| # --------------------------------------------------------- | |
| # Paso 2: Cargar el modelo base | |
| # --------------------------------------------------------- | |
| model_name = "meta-llama-3-8b" # Ajusta al modelo que tengas | |
| tokenizer = AutoTokenizer.from_pretrained(model_name) | |
| model = AutoModelForCausalLM.from_pretrained( | |
| model_name, | |
| load_in_8bit=True, # Para entrenamiento 8 bits (bitsandbytes) | |
| device_map="auto" | |
| ) | |
| # Preparar el modelo para entrenamiento en 8 bits | |
| model = prepare_model_for_int8_training(model) | |
| # --------------------------------------------------------- | |
| # Paso 3: Configurar LoRA | |
| # --------------------------------------------------------- | |
| lora_config = LoraConfig( | |
| r=8, | |
| lora_alpha=32, | |
| target_modules=["q_proj", "v_proj"], # T铆picos en Llama | |
| lora_dropout=0.05, | |
| bias="none", | |
| task_type="CAUSAL_LM" | |
| ) | |
| model = get_peft_model(model, lora_config) | |
| # --------------------------------------------------------- | |
| # Paso 4: Cargar y tokenizar el dataset | |
| # --------------------------------------------------------- | |
| # Si tu JSON est谩 local: | |
| dataset = load_dataset("json", data_files=dataset_path["train"], split="train") | |
| # Funci贸n de tokenizaci贸n | |
| def tokenize_fn(example): | |
| return tokenizer(example["text"], truncation=True, max_length=512) | |
| tokenized_dataset = dataset.map(tokenize_fn, batched=True) | |
| # --------------------------------------------------------- | |
| # Paso 5: Configuraci贸n de entrenamiento | |
| # --------------------------------------------------------- | |
| training_args = TrainingArguments( | |
| output_dir="llama3-lora-output", | |
| per_device_train_batch_size=1, | |
| gradient_accumulation_steps=4, | |
| num_train_epochs=1, # Ajusta seg煤n tu caso | |
| learning_rate=2e-4, | |
| fp16=True, # Half precision | |
| logging_steps=10, | |
| evaluation_strategy="no", # Cambia a "steps"/"epoch" si tienes validaci贸n | |
| save_strategy="epoch" | |
| ) | |
| # Crear el Trainer | |
| trainer = Trainer( | |
| model=model, | |
| args=training_args, | |
| train_dataset=tokenized_dataset, | |
| tokenizer=tokenizer | |
| ) | |
| # --------------------------------------------------------- | |
| # Paso 6: Entrenar y guardar | |
| # --------------------------------------------------------- | |
| trainer.train() | |
| model.save_pretrained("llama3-lora-output") | |
| print("Entrenamiento finalizado. Adaptadores guardados en 'llama3-lora-output'.") | |
| if __name__ == "__main__": | |
| main() | |