Download train_qwen3_codeforces.py from burtenshaw/training-scripts: direct link, hf CLI and curl.
- Browser
- Download file 2.52 kB
-
https://huggingface.co/burtenshaw/training-scripts/resolve/main/train_qwen3_codeforces.py
- Command line
-
hf download hf://burtenshaw/training-scripts/train_qwen3_codeforces.py
-
curl -L -o train_qwen3_codeforces.py https://huggingface.co/burtenshaw/training-scripts/resolve/main/train_qwen3_codeforces.py
2.52 kB
| # /// script | |
| # dependencies = [ | |
| # "trl>=0.18.0", | |
| # "peft>=0.7.0", | |
| # "transformers>=4.51.0", | |
| # "accelerate>=0.24.0", | |
| # "trackio", | |
| # "bitsandbytes", | |
| # ] | |
| # /// | |
| import trackio | |
| from datasets import load_dataset | |
| from peft import LoraConfig | |
| from trl import SFTTrainer, SFTConfig | |
| from transformers import AutoTokenizer | |
| # Load dataset | |
| print("π¦ Loading dataset...") | |
| dataset = load_dataset("open-r1/codeforces-cots", split="train") | |
| print(f"β Dataset loaded: {len(dataset)} examples") | |
| # Keep only the messages column (TRL SFT format) | |
| dataset = dataset.select_columns(["messages"]) | |
| print(f"β Kept only 'messages' column") | |
| # Create train/eval split | |
| print("π Creating train/eval split...") | |
| dataset_split = dataset.train_test_split(test_size=0.02, seed=42) | |
| train_dataset = dataset_split["train"] | |
| eval_dataset = dataset_split["test"] | |
| print(f" Train: {len(train_dataset)} examples") | |
| print(f" Eval: {len(eval_dataset)} examples") | |
| # Training configuration | |
| config = SFTConfig( | |
| # Hub settings | |
| output_dir="qwen3-codeforces-cots-sft", | |
| push_to_hub=True, | |
| hub_model_id="burtenshaw/qwen3-codeforces-cots-sft", | |
| hub_strategy="every_save", | |
| # Training parameters | |
| num_train_epochs=1, | |
| per_device_train_batch_size=2, | |
| gradient_accumulation_steps=8, | |
| learning_rate=2e-4, | |
| max_length=4096, | |
| # Logging & checkpointing | |
| logging_steps=25, | |
| save_strategy="steps", | |
| save_steps=500, | |
| save_total_limit=2, | |
| # Evaluation | |
| eval_strategy="steps", | |
| eval_steps=500, | |
| # Optimization | |
| warmup_ratio=0.05, | |
| lr_scheduler_type="cosine", | |
| bf16=True, | |
| gradient_checkpointing=True, | |
| # Monitoring | |
| report_to="trackio", | |
| project="codeforces-sft", | |
| run_name="qwen3-0.6b-codeforces-cots", | |
| ) | |
| # LoRA configuration | |
| peft_config = LoraConfig( | |
| r=32, | |
| lora_alpha=64, | |
| lora_dropout=0.05, | |
| bias="none", | |
| task_type="CAUSAL_LM", | |
| target_modules=["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"], | |
| ) | |
| # Initialize and train | |
| print("π― Initializing trainer...") | |
| trainer = SFTTrainer( | |
| model="Qwen/Qwen3-0.6B", | |
| train_dataset=train_dataset, | |
| eval_dataset=eval_dataset, | |
| args=config, | |
| peft_config=peft_config, | |
| ) | |
| print("π Starting training...") | |
| trainer.train() | |
| print("πΎ Pushing to Hub...") | |
| trainer.push_to_hub() | |
| print("β Complete! Model at: https://huggingface.co/burtenshaw/qwen3-codeforces-cots-sft") | |
| print("π View metrics at: https://huggingface.co/spaces/burtenshaw/trackio") | |