oExz commited on
Commit
772f77b
·
verified ·
1 Parent(s): 15685a3

Upload train_unsloth.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. train_unsloth.py +10 -0
train_unsloth.py CHANGED
@@ -40,6 +40,16 @@ model, tokenizer = FastLanguageModel.from_pretrained(
40
  trust_remote_code=True,
41
  )
42
 
 
 
 
 
 
 
 
 
 
 
43
  model = FastLanguageModel.get_peft_model(
44
  model,
45
  r=16,
 
40
  trust_remote_code=True,
41
  )
42
 
43
+ # Fix eos_token issue with Qwen3VLProcessor
44
+ if hasattr(tokenizer, "tokenizer"):
45
+ # tokenizer is a processor, fix the underlying tokenizer
46
+ tok = tokenizer.tokenizer
47
+ tok.eos_token = tok.decode([tok.eos_token_id])
48
+ tok.pad_token = tok.eos_token
49
+ else:
50
+ tokenizer.eos_token = tokenizer.decode([tokenizer.eos_token_id])
51
+ tokenizer.pad_token = tokenizer.eos_token
52
+
53
  model = FastLanguageModel.get_peft_model(
54
  model,
55
  r=16,