Upload train_unsloth.py with huggingface_hub
Browse files- train_unsloth.py +10 -0
train_unsloth.py
CHANGED
|
@@ -40,6 +40,16 @@ model, tokenizer = FastLanguageModel.from_pretrained(
|
|
| 40 |
trust_remote_code=True,
|
| 41 |
)
|
| 42 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 43 |
model = FastLanguageModel.get_peft_model(
|
| 44 |
model,
|
| 45 |
r=16,
|
|
|
|
| 40 |
trust_remote_code=True,
|
| 41 |
)
|
| 42 |
|
| 43 |
+
# Fix eos_token issue with Qwen3VLProcessor
|
| 44 |
+
if hasattr(tokenizer, "tokenizer"):
|
| 45 |
+
# tokenizer is a processor, fix the underlying tokenizer
|
| 46 |
+
tok = tokenizer.tokenizer
|
| 47 |
+
tok.eos_token = tok.decode([tok.eos_token_id])
|
| 48 |
+
tok.pad_token = tok.eos_token
|
| 49 |
+
else:
|
| 50 |
+
tokenizer.eos_token = tokenizer.decode([tokenizer.eos_token_id])
|
| 51 |
+
tokenizer.pad_token = tokenizer.eos_token
|
| 52 |
+
|
| 53 |
model = FastLanguageModel.get_peft_model(
|
| 54 |
model,
|
| 55 |
r=16,
|