add model

generate.py

@@ -11,16 +11,6 @@ MAX_MAX_NEW_TOKENS = 2048
 DEFAULT_MAX_NEW_TOKENS = 1024
 MAX_INPUT_TOKEN_LENGTH = int(os.getenv("MAX_INPUT_TOKEN_LENGTH", "4096"))
 
-if not torch.cuda.is_available():
-    DESCRIPTION += "\n<p>Running on CPU 🥶 This demo does not work on CPU.</p>"
-
-
-if torch.cuda.is_available():
-    model_id = "meta-llama/Llama-2-13b-chat-hf"
-    model = AutoModelForCausalLM.from_pretrained(model_id, device_map="auto", load_in_4bit=True)
-    tokenizer = AutoTokenizer.from_pretrained(model_id)
-    tokenizer.use_default_system_prompt = False
-
 # talk to llm
 # @spaces.GPU
 def generate(
@@ -33,6 +23,12 @@ def generate(
     top_k: int = 50,
     repetition_penalty: float = 1.2,
 ) -> Iterator[str]:
+    
+    model_id = "meta-llama/Llama-2-13b-chat-hf"
+    model = AutoModelForCausalLM.from_pretrained(model_id, device_map="auto", load_in_4bit=True)
+    tokenizer = AutoTokenizer.from_pretrained(model_id)
+    tokenizer.use_default_system_prompt = False
+    
     conversation = []
     if system_prompt:
         conversation.append({"role": "system", "content": system_prompt})