update

examples/tutorials/dpo/ultrachat/requirements.txt

@@ -1,4 +1,4 @@
 datasets
 transformers==4.50.0
 trl==0.16.1
-deepspeed==0.18.6
+deepspeed==0.15.2

examples/tutorials/dpo/ultrachat/step_2_train_sft_model_ddp.py

@@ -4,7 +4,7 @@
 https://huggingface.co/docs/trl/v0.16.1/en/sft_trainer
 
 双卡 V00 32G 全参微调
-torchrun --nproc_per_node=2 step_2_train_sft_model_stages_1.py
+torchrun --nproc_per_node=2 step_2_train_sft_model_ddp.py
 
 """
 import argparse
@@ -59,7 +59,7 @@ def get_args():
     ),
     parser.add_argument(
         "--output_model_dir",
-        default=(temp_directory / "trained_models/qwen2_5-0_5B-ultrachat-sft").as_posix(),
+        default=(temp_directory / "trained_models/qwen2_5-0_5B-ultrachat-sft-ddp").as_posix(),
         type=str
     ),
     parser.add_argument(
@@ -92,8 +92,16 @@ def format_func(examples, tokenizer):
 def main():
     args = get_args()
 
-    model = AutoModelForCausalLM.from_pretrained(args.model_name)
-    tokenizer = AutoTokenizer.from_pretrained(args.model_name)
+    model = AutoModelForCausalLM.from_pretrained(
+        args.model_name,
+        cache_dir=args.model_cache_dir,
+        trust_remote_code=True,
+    )
+    tokenizer = AutoTokenizer.from_pretrained(
+        args.model_name,
+        cache_dir=args.model_cache_dir,
+        trust_remote_code=True,
+    )
     if tokenizer.pad_token is None:
         tokenizer.pad_token = tokenizer.eos_token
         tokenizer.pad_token_id = tokenizer.eos_token_id

examples/tutorials/dpo/ultrachat/step_2_train_sft_model_deepspeed.py

@@ -99,14 +99,14 @@ def main():
 
     model = AutoModelForCausalLM.from_pretrained(
         args.model_name,
-        # cache_dir=args.model_cache_dir,
+        cache_dir=args.model_cache_dir,
         torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32,
         low_cpu_mem_usage=True,
         trust_remote_code=True,
     )
     tokenizer = AutoTokenizer.from_pretrained(
         args.model_name,
-        # cache_dir=args.model_cache_dir,
+        cache_dir=args.model_cache_dir,
         trust_remote_code=True,
     )
     if tokenizer.pad_token is None:
@@ -132,49 +132,30 @@ def main():
 
     response_template = "<|im_end|>\n<|im_start|>assistant\n"
     instruction_template = "<|im_end|>\n<|im_start|>user\n"
-    # data_collator = DataCollatorForCompletionOnlyLM(
-    #     response_template=response_template,
-    #     instruction_template=instruction_template,
-    #     tokenizer=tokenizer,
-    #     mlm=False,
-    # )
-    data_collator = DataCollatorForLanguageModeling(
+    data_collator = DataCollatorForCompletionOnlyLM(
+        response_template=response_template,
+        instruction_template=instruction_template,
         tokenizer=tokenizer,
         mlm=False,
     )
 
     sft_config = SFTConfig(
         output_dir=args.output_model_dir,
-        deepspeed="./ds_config/deepspeed_stage_3_config.json",  # 添加deepspeed配置文件
+        dataset_text_field="formated_text",
+        deepspeed="./ds_config/deepspeed_stage_3_config.json",
         num_train_epochs=1,
         per_device_train_batch_size=1 if debug_mode else 2,
         gradient_accumulation_steps=1 if debug_mode else 8,
-        save_strategy="steps",
-        save_steps=100,
-        save_total_limit=2,
-        logging_steps=100,
-        learning_rate=2e-5,
         warmup_steps=100,
+        learning_rate=3e-5,
+        logging_steps=100,
+        save_steps=10,
+        save_total_limit=2,
+        optim="adamw_8bit",
+        weight_decay=0,
         lr_scheduler_type="cosine",
-        fp16=True if torch.cuda.is_available() else False,
-        bf16=False,
-        gradient_checkpointing=True,
-        optim="adamw_torch",
+        seed=3407,
         report_to="none",
-        max_seq_length=1024 if debug_mode else 2048,
-        packing=False,
-        dataloader_num_workers=args.num_workers if args.num_workers else 2,
-        dataloader_pin_memory=False,
-        remove_unused_columns=True,
-        ddp_find_unused_parameters=False,  # 重要
-        group_by_length=False,
-        no_cuda=False,
-        seed=42,
-        data_seed=42,
-        local_rank=args.local_rank,  # 使用local_rank
-        ddp_backend="nccl" if torch.cuda.is_available() else "gloo",
-        # 关键修改7：添加deepspeed特定配置
-        torch_compile=False,  # 禁用torch.compile避免冲突
     )
 
     trainer = SFTTrainer(
@@ -184,11 +165,9 @@ def main():
         train_dataset=train_dataset,
     )
 
-    # 开始训练
     print("开始训练...")
     trainer.train()
 
-    # 保存模型
     if args.local_rank in (-1, 0):
         print(f"保存模型到: {args.output_model_dir}")
         trainer.save_model()

examples/tutorials/dpo/ultrachat/{step_2_train_sft_model_stages_0.py → step_2_train_sft_model_single_gpu.py}

@@ -4,6 +4,8 @@
 https://huggingface.co/docs/trl/v0.16.1/en/sft_trainer
 
 单卡 V00 32G 全参微调
+python3 step_2_train_sft_model_single_gpu.py
+
 """
 import argparse
 import os
@@ -11,8 +13,6 @@ from pathlib import Path
 import platform
 
 os.environ["HF_ENDPOINT"] = "https://hf-mirror.com"
-os.environ["CUDA_VISIBLE_DEVICES"] = "0,1"  # 指定使用2张卡
-os.environ["TOKENIZERS_PARALLELISM"] = "false"
 
 debug_mode = True if platform.system() in ("Windows", "Darwin") else False
 print(f"debug_mode: {debug_mode}")
@@ -55,7 +55,7 @@ def get_args():
     ),
     parser.add_argument(
         "--output_model_dir",
-        default=(temp_directory / "trained_models/qwen2_5-0_5B-ultrachat-sft").as_posix(),
+        default=(temp_directory / "trained_models/qwen2_5-0_5B-ultrachat-sft-single-gpu").as_posix(),
         type=str
     ),
     parser.add_argument(
@@ -90,9 +90,17 @@ def main():
 
     device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 
-    model = AutoModelForCausalLM.from_pretrained(args.model_name)
+    model = AutoModelForCausalLM.from_pretrained(
+        args.model_name,
+        cache_dir=args.model_cache_dir,
+        trust_remote_code=True,
+    )
+    tokenizer = AutoTokenizer.from_pretrained(
+        args.model_name,
+        cache_dir=args.model_cache_dir,
+        trust_remote_code=True,
+    )
     model = model.to(device)
-    tokenizer = AutoTokenizer.from_pretrained(args.model_name)
     if tokenizer.pad_token is None:
         tokenizer.pad_token = tokenizer.eos_token
         tokenizer.pad_token_id = tokenizer.eos_token_id
@@ -136,6 +144,7 @@ def main():
         optim="adamw_torch",
         report_to="none",
         max_length=1024 if debug_mode else 2048,
+        dataloader_pin_memory=False # 为 True时，内存中的数据不会被降级到磁盘，只有当数据真的在内存中才可被GPU高效访问。
     )
 
     trainer = SFTTrainer(

examples/tutorials/rlhf/gpt2_sst2/step_5_ppo_rlhf.py

@@ -1,5 +1,8 @@
 #!/usr/bin/python3
 # -*- coding: utf-8 -*-
+"""
+https://huggingface.co/docs/trl/v0.16.1/en/ppo_trainer
+"""
 import argparse
 import copy
 import os