| ``` |
| model, tokenizer = FastLanguageModel.from_pretrained( |
| model_name = "unsloth/GLM-4.7-Flash", |
| max_seq_length = 2048, # Choose any for long context! |
| load_in_4bit = False, # 4 bit quantization to reduce memory |
| load_in_8bit = False, # [NEW!] A bit more accurate, uses 2x memory |
| full_finetuning = False, # [NEW!] We have full finetuning now! |
| trust_remote_code = True, |
| unsloth_force_compile = False, |
| ) |
| |
| model = FastLanguageModel.get_peft_model( |
| model, |
| r = 8, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128 |
| target_modules = ["q_proj", "k_proj", "v_proj", "o_proj", |
| "gate_proj", "up_proj", "down_proj", |
| "in_proj", "out_proj",], |
| lora_alpha = 16, |
| lora_dropout = 0, # Supports any, but = 0 is optimized |
| bias = "none", # Supports any, but = "none" is optimized |
| # [NEW] "unsloth" uses 30% less VRAM, fits 2x larger batch sizes! |
| use_gradient_checkpointing = "unsloth", # True or "unsloth" for very long context |
| random_state = 3407, |
| use_rslora = False, # We support rank stabilized LoRA |
| loftq_config = None, # And LoftQ |
| ) |
| |
| dataset = load_dataset("unsloth/OpenMathReasoning-mini", split = "cot") |
| |
| # This step might take ~3m on this A100 notebook |
| from trl import SFTTrainer, SFTConfig |
| trainer = SFTTrainer( |
| model = model, |
| tokenizer = tokenizer, |
| train_dataset = dataset, |
| eval_dataset = None, # Can set up evaluation! |
| args = SFTConfig( |
| dataset_text_field = "text", |
| dataset_num_proc=1, # Increasing "might" throw error on Colab/other envs. |
| per_device_train_batch_size = 4, |
| gradient_accumulation_steps = 2, # Use GA to mimic batch size! |
| warmup_steps = 5, |
| # num_train_epochs = 1, # Set this for 1 full training run. |
| max_steps = 60, |
| learning_rate = 2e-4, # Reduce to 2e-5 for long training runs |
| logging_steps = 1, |
| optim = "adamw_8bit", |
| weight_decay = 0.001, |
| lr_scheduler_type = "linear", |
| seed = 3407, |
| report_to = "none", # Use TrackIO/WandB etc |
| ), |
| ) |
| |
| trainer = train_on_responses_only( |
| trainer, |
| instruction_part = "[gMASK]<sop><|user|>", # Updated for GLM |
| response_part = "<|assistant|><think>", |
| ) |
| |
| |
