phxdev commited on
Commit
26d8f5c
Β·
verified Β·
1 Parent(s): 86a58d0

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +51 -18
app.py CHANGED
@@ -66,6 +66,11 @@ class CreedBrattonAI:
66
  padding_side="left"
67
  )
68
 
 
 
 
 
 
69
  # Add Creed's custom tokens
70
  custom_tokens = ["<thinking>", "<conspiracy>", "<tangent>"]
71
  print(f"🎸 Adding Creed's custom tokens: {custom_tokens}")
@@ -73,9 +78,6 @@ class CreedBrattonAI:
73
  num_added_tokens = self.tokenizer.add_tokens(custom_tokens)
74
  print(f"βœ… Added {num_added_tokens} custom tokens")
75
 
76
- if self.tokenizer.pad_token is None:
77
- self.tokenizer.pad_token = self.tokenizer.eos_token
78
-
79
  print(f"πŸ€– Loading model on {self.device}...")
80
 
81
  # Load model with proper device handling
@@ -87,6 +89,10 @@ class CreedBrattonAI:
87
  trust_remote_code=True,
88
  low_cpu_mem_usage=True
89
  )
 
 
 
 
90
  else:
91
  self.model = AutoModelForCausalLM.from_pretrained(
92
  model_name,
@@ -104,12 +110,16 @@ class CreedBrattonAI:
104
 
105
  self.model.eval()
106
 
 
 
 
 
107
  self.model_loaded = True
108
  self.loading = False
109
- print(f"βœ… Creed's consciousness loaded on {self.device}!")
110
 
111
  # GPU memory info
112
- if self.device == "cuda":
113
  print(f"πŸ”₯ GPU Memory Used: {torch.cuda.memory_allocated() // 1024**2} MB")
114
  print(f"πŸ“Š GPU Memory Cached: {torch.cuda.memory_reserved() // 1024**2} MB")
115
 
@@ -120,12 +130,18 @@ class CreedBrattonAI:
120
  base_model = "Qwen/Qwen2.5-0.5B-Instruct"
121
  self.tokenizer = AutoTokenizer.from_pretrained(base_model)
122
 
 
 
 
 
123
  if self.device == "cuda":
124
  self.model = AutoModelForCausalLM.from_pretrained(
125
  base_model,
126
  torch_dtype=torch.float16,
127
  device_map="auto"
128
  )
 
 
129
  else:
130
  self.model = AutoModelForCausalLM.from_pretrained(
131
  base_model,
@@ -136,31 +152,46 @@ class CreedBrattonAI:
136
 
137
  self.model.eval()
138
  self.model_loaded = True
139
- print(f"βœ… Fallback model loaded on {self.device}")
 
140
  except Exception as fallback_error:
141
  print(f"❌ Fallback also failed: {fallback_error}")
142
  self.loading = False
143
 
144
  @spaces.GPU if SPACES_AVAILABLE else lambda func: func
145
  def generate_response_gpu(self, conversation: str) -> str:
146
- """Generate response using the loaded model (already on correct device)"""
147
 
148
  if not self.model_loaded:
149
  return "❌ Model not loaded"
150
 
151
  try:
152
- # Model is already on the correct device from load_model()
153
- # No need to move it around anymore
 
 
154
 
155
- # Tokenize input and put on same device as model
156
- inputs = self.tokenizer.encode(conversation, return_tensors="pt")
157
- if self.device == "cuda":
158
- inputs = inputs.to(self.device)
 
 
 
 
 
 
 
 
 
 
 
159
 
160
- # Generate response
161
  with torch.no_grad():
162
  outputs = self.model.generate(
163
- inputs,
 
164
  max_new_tokens=200,
165
  do_sample=True,
166
  temperature=0.9,
@@ -175,11 +206,12 @@ class CreedBrattonAI:
175
 
176
  # Decode response
177
  full_response = self.tokenizer.decode(outputs[0], skip_special_tokens=True)
178
- response = full_response[len(self.tokenizer.decode(inputs[0], skip_special_tokens=True)):].strip()
179
 
180
  return self._clean_response(response)
181
 
182
  except Exception as e:
 
183
  return f"🎸 *Creed scratches his head* Something weird happened... {str(e)[:100]}"
184
 
185
  def generate_response(self, message: str, history: List[Dict]) -> Iterator[str]:
@@ -357,6 +389,7 @@ def main():
357
 
358
  # Memory status if GPU available
359
  if torch.cuda.is_available() and creed_ai.model_loaded:
 
360
  print(f"πŸ”₯ Final GPU Memory: {torch.cuda.memory_allocated() // 1024**2} MB allocated")
361
  print(f"πŸ“Š GPU Memory Reserved: {torch.cuda.memory_reserved() // 1024**2} MB reserved")
362
 
@@ -796,14 +829,14 @@ def main():
796
  </div>
797
  """)
798
 
799
- # Launch with modern styling
800
  print("πŸš€ Launching Real Creed AI with modern glassmorphism design...")
801
 
802
  demo.launch(
803
  ssr_mode=False,
804
  server_name="0.0.0.0",
805
  server_port=7860,
806
- share=False,
807
  show_error=True
808
  )
809
 
 
66
  padding_side="left"
67
  )
68
 
69
+ # Fix pad token issue
70
+ if self.tokenizer.pad_token is None:
71
+ self.tokenizer.pad_token = self.tokenizer.unk_token or "[PAD]"
72
+ print(f"πŸ”§ Set pad_token to: {self.tokenizer.pad_token}")
73
+
74
  # Add Creed's custom tokens
75
  custom_tokens = ["<thinking>", "<conspiracy>", "<tangent>"]
76
  print(f"🎸 Adding Creed's custom tokens: {custom_tokens}")
 
78
  num_added_tokens = self.tokenizer.add_tokens(custom_tokens)
79
  print(f"βœ… Added {num_added_tokens} custom tokens")
80
 
 
 
 
81
  print(f"πŸ€– Loading model on {self.device}...")
82
 
83
  # Load model with proper device handling
 
89
  trust_remote_code=True,
90
  low_cpu_mem_usage=True
91
  )
92
+ # Explicitly ensure model is on CUDA
93
+ if self.model.device.type != "cuda":
94
+ print("πŸ”§ Explicitly moving model to CUDA...")
95
+ self.model = self.model.to(self.device)
96
  else:
97
  self.model = AutoModelForCausalLM.from_pretrained(
98
  model_name,
 
110
 
111
  self.model.eval()
112
 
113
+ # Verify device placement
114
+ model_device = next(self.model.parameters()).device
115
+ print(f"🎯 Model is actually on: {model_device}")
116
+
117
  self.model_loaded = True
118
  self.loading = False
119
+ print(f"βœ… Creed's consciousness loaded on {model_device}!")
120
 
121
  # GPU memory info
122
+ if self.device == "cuda" and torch.cuda.is_available():
123
  print(f"πŸ”₯ GPU Memory Used: {torch.cuda.memory_allocated() // 1024**2} MB")
124
  print(f"πŸ“Š GPU Memory Cached: {torch.cuda.memory_reserved() // 1024**2} MB")
125
 
 
130
  base_model = "Qwen/Qwen2.5-0.5B-Instruct"
131
  self.tokenizer = AutoTokenizer.from_pretrained(base_model)
132
 
133
+ # Fix pad token for fallback too
134
+ if self.tokenizer.pad_token is None:
135
+ self.tokenizer.pad_token = self.tokenizer.unk_token or "[PAD]"
136
+
137
  if self.device == "cuda":
138
  self.model = AutoModelForCausalLM.from_pretrained(
139
  base_model,
140
  torch_dtype=torch.float16,
141
  device_map="auto"
142
  )
143
+ if self.model.device.type != "cuda":
144
+ self.model = self.model.to(self.device)
145
  else:
146
  self.model = AutoModelForCausalLM.from_pretrained(
147
  base_model,
 
152
 
153
  self.model.eval()
154
  self.model_loaded = True
155
+ model_device = next(self.model.parameters()).device
156
+ print(f"βœ… Fallback model loaded on {model_device}")
157
  except Exception as fallback_error:
158
  print(f"❌ Fallback also failed: {fallback_error}")
159
  self.loading = False
160
 
161
  @spaces.GPU if SPACES_AVAILABLE else lambda func: func
162
  def generate_response_gpu(self, conversation: str) -> str:
163
+ """Generate response using the loaded model with proper device handling"""
164
 
165
  if not self.model_loaded:
166
  return "❌ Model not loaded"
167
 
168
  try:
169
+ # Ensure model is on the correct device
170
+ if self.device == "cuda" and self.model.device.type != "cuda":
171
+ print(f"πŸ”„ Moving model from {self.model.device} to {self.device}")
172
+ self.model = self.model.to(self.device)
173
 
174
+ # Tokenize input with attention mask
175
+ inputs = self.tokenizer(
176
+ conversation,
177
+ return_tensors="pt",
178
+ padding=True,
179
+ truncation=True,
180
+ max_length=2048,
181
+ add_special_tokens=True
182
+ )
183
+
184
+ # Move inputs to same device as model
185
+ input_ids = inputs['input_ids'].to(self.device)
186
+ attention_mask = inputs['attention_mask'].to(self.device)
187
+
188
+ print(f"πŸ” Model device: {self.model.device}, Input device: {input_ids.device}")
189
 
190
+ # Generate response with proper attention mask
191
  with torch.no_grad():
192
  outputs = self.model.generate(
193
+ input_ids=input_ids,
194
+ attention_mask=attention_mask, # Pass attention mask
195
  max_new_tokens=200,
196
  do_sample=True,
197
  temperature=0.9,
 
206
 
207
  # Decode response
208
  full_response = self.tokenizer.decode(outputs[0], skip_special_tokens=True)
209
+ response = full_response[len(self.tokenizer.decode(input_ids[0], skip_special_tokens=True)):].strip()
210
 
211
  return self._clean_response(response)
212
 
213
  except Exception as e:
214
+ print(f"❌ Generation error: {e}")
215
  return f"🎸 *Creed scratches his head* Something weird happened... {str(e)[:100]}"
216
 
217
  def generate_response(self, message: str, history: List[Dict]) -> Iterator[str]:
 
389
 
390
  # Memory status if GPU available
391
  if torch.cuda.is_available() and creed_ai.model_loaded:
392
+ print(f"🎯 Model device verification: {next(creed_ai.model.parameters()).device}")
393
  print(f"πŸ”₯ Final GPU Memory: {torch.cuda.memory_allocated() // 1024**2} MB allocated")
394
  print(f"πŸ“Š GPU Memory Reserved: {torch.cuda.memory_reserved() // 1024**2} MB reserved")
395
 
 
829
  </div>
830
  """)
831
 
832
+ # Launch with modern styling and public sharing
833
  print("πŸš€ Launching Real Creed AI with modern glassmorphism design...")
834
 
835
  demo.launch(
836
  ssr_mode=False,
837
  server_name="0.0.0.0",
838
  server_port=7860,
839
+ share=True, # Create public link
840
  show_error=True
841
  )
842