Upload train_unsloth.py with huggingface_hub

train_unsloth.py

@@ -15,6 +15,7 @@ import os
 from datasets import load_dataset
 from trl import SFTTrainer, SFTConfig
 from unsloth import FastLanguageModel
+from transformers import AutoTokenizer
 import trackio
 
 hf_token = os.getenv("HF_TOKEN")
@@ -32,7 +33,7 @@ eval_ds = dataset["eval"]
 print(f"Train: {len(train_ds)}, Eval: {len(eval_ds)}")
 
 print(f"Loading base model {BASE_MODEL} with Unsloth...")
-model, tokenizer = FastLanguageModel.from_pretrained(
+model, _ = FastLanguageModel.from_pretrained(
     model_name=BASE_MODEL,
     max_seq_length=1024,
     dtype=None,
@@ -40,15 +41,9 @@ model, tokenizer = FastLanguageModel.from_pretrained(
     trust_remote_code=True,
 )
 
-# Fix eos_token issue with Qwen3VLProcessor
-if hasattr(tokenizer, "tokenizer"):
-    # tokenizer is a processor, fix the underlying tokenizer
-    tok = tokenizer.tokenizer
-    tok.eos_token = tok.decode([tok.eos_token_id])
-    tok.pad_token = tok.eos_token
-else:
-    tokenizer.eos_token = tokenizer.decode([tokenizer.eos_token_id])
-    tokenizer.pad_token = tokenizer.eos_token
+# Load tokenizer directly to avoid Qwen3VLProcessor eos_token issue
+tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL, trust_remote_code=True)
+tokenizer.pad_token = tokenizer.eos_token
 
 model = FastLanguageModel.get_peft_model(
     model,