Spaces:
Sleeping
Sleeping
Download train.py from elrplays/peter-training: direct link, hf CLI and curl.
- Browser
- Download file 1.81 kB
-
https://huggingface.co/spaces/elrplays/peter-training/resolve/main/train.py
- Command line
-
hf download hf://spaces/elrplays/peter-training/train.py
-
curl -L -o train.py https://huggingface.co/spaces/elrplays/peter-training/resolve/main/train.py
1.81 kB
| from transformers import AutoModelForCausalLM, AutoTokenizer, TrainingArguments | |
| from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training | |
| from datasets import load_dataset | |
| from trl import SFTConfig, SFTTrainer | |
| import torch | |
| # 1. Configurações | |
| MODEL_NAME = "Qwen/Qwen2.5-Coder-7B-Instruct" | |
| # 2. Carregar Tokenizer e Modelo (Modo Padrão HF) | |
| tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME) | |
| tokenizer.pad_token = tokenizer.eos_token | |
| model = AutoModelForCausalLM.from_pretrained( | |
| MODEL_NAME, | |
| load_in_4bit=True, # Quantização 4-bit padrão bitsandbytes | |
| device_map="auto", | |
| torch_dtype=torch.float16 | |
| ) | |
| # 3. Preparar LoRA sem depender do Unsloth | |
| model = prepare_model_for_kbit_training(model) | |
| peft_config = LoraConfig( | |
| r=16, | |
| lora_alpha=16, | |
| lora_dropout=0, | |
| target_modules=["q_proj", "k_proj", "v_proj", "o_proj"], | |
| task_type="CAUSAL_LM", | |
| ) | |
| model = get_peft_model(model, peft_config) | |
| # 4. Dataset | |
| dataset = load_dataset("json", data_files="seu_dataset.jsonl", split="train") | |
| # 5. SFTConfig (O jeito novo que evita o erro de 'tokenizer') | |
| sft_config = SFTConfig( | |
| output_dir="./peter-lora", | |
| max_seq_length=1024, # Reduzi de 2048 para não explodir a RAM do Space | |
| dataset_text_field="text", | |
| per_device_train_batch_size=1, | |
| gradient_accumulation_steps=4, | |
| learning_rate=2e-4, | |
| logging_steps=1, | |
| fp16=True, | |
| save_strategy="no", | |
| push_to_hub=False | |
| ) | |
| # 6. Trainer Corrigido | |
| trainer = SFTTrainer( | |
| model=model, | |
| train_dataset=dataset, | |
| args=sft_config, | |
| peft_config=peft_config, | |
| # Removido o argumento 'tokenizer' daqui para o SFTConfig assumir | |
| ) | |
| trainer.train() | |
| # 7. Salvar e Enviar | |
| model.push_to_hub("seuusername/peter-lora", private=True) | |
| tokenizer.push_to_hub("seuusername/peter-lora", private=True) |