Update README.md

README.md

@@ -24,13 +24,13 @@ The following `bitsandbytes` quantization config was used during training:
 ```python
 from peft import PeftModel, PeftConfig
 from huggingface_hub import login
-from transformers import AutoModelForCausalLM, AutoTokenizer
+from transformers import AutoModelForCausalLM, AutoTokenizer, AddedToken
 
 login("[YOUR HF TOKEN HERE FOR USING LLAMA]")
 config = PeftConfig.from_pretrained("ChangeIsKey/llama-7b-lexical-substitution")
 base_model = AutoModelForCausalLM.from_pretrained("meta-llama/Llama-2-7b-hf")
 
-tokenizer = AutoTokenizer.from_pretrained(base_model_name, use_fast=False, trust_remote_code=True, cache_dir=LLMs_CACHE_DIR)
+tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-hf", use_fast=False, trust_remote_code=True)
 tokenizer.add_special_tokens({ "additional_special_tokens":[AddedToken("<|s|>"), AddedToken("<|answer|>"), AddedToken("<|end|>")]})
 if tokenizer.pad_token is None:
     tokenizer.add_special_tokens({'pad_token': '[PAD]'})