Upload train_unsloth.py with huggingface_hub
Browse files- train_unsloth.py +2 -1
train_unsloth.py
CHANGED
|
@@ -41,8 +41,9 @@ model, _ = FastLanguageModel.from_pretrained(
|
|
| 41 |
trust_remote_code=True,
|
| 42 |
)
|
| 43 |
|
| 44 |
-
# Load tokenizer directly
|
| 45 |
tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL, trust_remote_code=True)
|
|
|
|
| 46 |
tokenizer.pad_token = tokenizer.eos_token
|
| 47 |
|
| 48 |
model = FastLanguageModel.get_peft_model(
|
|
|
|
| 41 |
trust_remote_code=True,
|
| 42 |
)
|
| 43 |
|
| 44 |
+
# Load tokenizer directly and fix eos_token
|
| 45 |
tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL, trust_remote_code=True)
|
| 46 |
+
tokenizer.eos_token = "<|im_end|>"
|
| 47 |
tokenizer.pad_token = tokenizer.eos_token
|
| 48 |
|
| 49 |
model = FastLanguageModel.get_peft_model(
|