Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
|
@@ -66,6 +66,11 @@ class CreedBrattonAI:
|
|
| 66 |
padding_side="left"
|
| 67 |
)
|
| 68 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 69 |
# Add Creed's custom tokens
|
| 70 |
custom_tokens = ["<thinking>", "<conspiracy>", "<tangent>"]
|
| 71 |
print(f"πΈ Adding Creed's custom tokens: {custom_tokens}")
|
|
@@ -73,9 +78,6 @@ class CreedBrattonAI:
|
|
| 73 |
num_added_tokens = self.tokenizer.add_tokens(custom_tokens)
|
| 74 |
print(f"β
Added {num_added_tokens} custom tokens")
|
| 75 |
|
| 76 |
-
if self.tokenizer.pad_token is None:
|
| 77 |
-
self.tokenizer.pad_token = self.tokenizer.eos_token
|
| 78 |
-
|
| 79 |
print(f"π€ Loading model on {self.device}...")
|
| 80 |
|
| 81 |
# Load model with proper device handling
|
|
@@ -87,6 +89,10 @@ class CreedBrattonAI:
|
|
| 87 |
trust_remote_code=True,
|
| 88 |
low_cpu_mem_usage=True
|
| 89 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 90 |
else:
|
| 91 |
self.model = AutoModelForCausalLM.from_pretrained(
|
| 92 |
model_name,
|
|
@@ -104,12 +110,16 @@ class CreedBrattonAI:
|
|
| 104 |
|
| 105 |
self.model.eval()
|
| 106 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 107 |
self.model_loaded = True
|
| 108 |
self.loading = False
|
| 109 |
-
print(f"β
Creed's consciousness loaded on {
|
| 110 |
|
| 111 |
# GPU memory info
|
| 112 |
-
if self.device == "cuda":
|
| 113 |
print(f"π₯ GPU Memory Used: {torch.cuda.memory_allocated() // 1024**2} MB")
|
| 114 |
print(f"π GPU Memory Cached: {torch.cuda.memory_reserved() // 1024**2} MB")
|
| 115 |
|
|
@@ -120,12 +130,18 @@ class CreedBrattonAI:
|
|
| 120 |
base_model = "Qwen/Qwen2.5-0.5B-Instruct"
|
| 121 |
self.tokenizer = AutoTokenizer.from_pretrained(base_model)
|
| 122 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 123 |
if self.device == "cuda":
|
| 124 |
self.model = AutoModelForCausalLM.from_pretrained(
|
| 125 |
base_model,
|
| 126 |
torch_dtype=torch.float16,
|
| 127 |
device_map="auto"
|
| 128 |
)
|
|
|
|
|
|
|
| 129 |
else:
|
| 130 |
self.model = AutoModelForCausalLM.from_pretrained(
|
| 131 |
base_model,
|
|
@@ -136,31 +152,46 @@ class CreedBrattonAI:
|
|
| 136 |
|
| 137 |
self.model.eval()
|
| 138 |
self.model_loaded = True
|
| 139 |
-
|
|
|
|
| 140 |
except Exception as fallback_error:
|
| 141 |
print(f"β Fallback also failed: {fallback_error}")
|
| 142 |
self.loading = False
|
| 143 |
|
| 144 |
@spaces.GPU if SPACES_AVAILABLE else lambda func: func
|
| 145 |
def generate_response_gpu(self, conversation: str) -> str:
|
| 146 |
-
"""Generate response using the loaded model
|
| 147 |
|
| 148 |
if not self.model_loaded:
|
| 149 |
return "β Model not loaded"
|
| 150 |
|
| 151 |
try:
|
| 152 |
-
#
|
| 153 |
-
|
|
|
|
|
|
|
| 154 |
|
| 155 |
-
# Tokenize input
|
| 156 |
-
inputs = self.tokenizer
|
| 157 |
-
|
| 158 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 159 |
|
| 160 |
-
# Generate response
|
| 161 |
with torch.no_grad():
|
| 162 |
outputs = self.model.generate(
|
| 163 |
-
|
|
|
|
| 164 |
max_new_tokens=200,
|
| 165 |
do_sample=True,
|
| 166 |
temperature=0.9,
|
|
@@ -175,11 +206,12 @@ class CreedBrattonAI:
|
|
| 175 |
|
| 176 |
# Decode response
|
| 177 |
full_response = self.tokenizer.decode(outputs[0], skip_special_tokens=True)
|
| 178 |
-
response = full_response[len(self.tokenizer.decode(
|
| 179 |
|
| 180 |
return self._clean_response(response)
|
| 181 |
|
| 182 |
except Exception as e:
|
|
|
|
| 183 |
return f"πΈ *Creed scratches his head* Something weird happened... {str(e)[:100]}"
|
| 184 |
|
| 185 |
def generate_response(self, message: str, history: List[Dict]) -> Iterator[str]:
|
|
@@ -357,6 +389,7 @@ def main():
|
|
| 357 |
|
| 358 |
# Memory status if GPU available
|
| 359 |
if torch.cuda.is_available() and creed_ai.model_loaded:
|
|
|
|
| 360 |
print(f"π₯ Final GPU Memory: {torch.cuda.memory_allocated() // 1024**2} MB allocated")
|
| 361 |
print(f"π GPU Memory Reserved: {torch.cuda.memory_reserved() // 1024**2} MB reserved")
|
| 362 |
|
|
@@ -796,14 +829,14 @@ def main():
|
|
| 796 |
</div>
|
| 797 |
""")
|
| 798 |
|
| 799 |
-
# Launch with modern styling
|
| 800 |
print("π Launching Real Creed AI with modern glassmorphism design...")
|
| 801 |
|
| 802 |
demo.launch(
|
| 803 |
ssr_mode=False,
|
| 804 |
server_name="0.0.0.0",
|
| 805 |
server_port=7860,
|
| 806 |
-
share=
|
| 807 |
show_error=True
|
| 808 |
)
|
| 809 |
|
|
|
|
| 66 |
padding_side="left"
|
| 67 |
)
|
| 68 |
|
| 69 |
+
# Fix pad token issue
|
| 70 |
+
if self.tokenizer.pad_token is None:
|
| 71 |
+
self.tokenizer.pad_token = self.tokenizer.unk_token or "[PAD]"
|
| 72 |
+
print(f"π§ Set pad_token to: {self.tokenizer.pad_token}")
|
| 73 |
+
|
| 74 |
# Add Creed's custom tokens
|
| 75 |
custom_tokens = ["<thinking>", "<conspiracy>", "<tangent>"]
|
| 76 |
print(f"πΈ Adding Creed's custom tokens: {custom_tokens}")
|
|
|
|
| 78 |
num_added_tokens = self.tokenizer.add_tokens(custom_tokens)
|
| 79 |
print(f"β
Added {num_added_tokens} custom tokens")
|
| 80 |
|
|
|
|
|
|
|
|
|
|
| 81 |
print(f"π€ Loading model on {self.device}...")
|
| 82 |
|
| 83 |
# Load model with proper device handling
|
|
|
|
| 89 |
trust_remote_code=True,
|
| 90 |
low_cpu_mem_usage=True
|
| 91 |
)
|
| 92 |
+
# Explicitly ensure model is on CUDA
|
| 93 |
+
if self.model.device.type != "cuda":
|
| 94 |
+
print("π§ Explicitly moving model to CUDA...")
|
| 95 |
+
self.model = self.model.to(self.device)
|
| 96 |
else:
|
| 97 |
self.model = AutoModelForCausalLM.from_pretrained(
|
| 98 |
model_name,
|
|
|
|
| 110 |
|
| 111 |
self.model.eval()
|
| 112 |
|
| 113 |
+
# Verify device placement
|
| 114 |
+
model_device = next(self.model.parameters()).device
|
| 115 |
+
print(f"π― Model is actually on: {model_device}")
|
| 116 |
+
|
| 117 |
self.model_loaded = True
|
| 118 |
self.loading = False
|
| 119 |
+
print(f"β
Creed's consciousness loaded on {model_device}!")
|
| 120 |
|
| 121 |
# GPU memory info
|
| 122 |
+
if self.device == "cuda" and torch.cuda.is_available():
|
| 123 |
print(f"π₯ GPU Memory Used: {torch.cuda.memory_allocated() // 1024**2} MB")
|
| 124 |
print(f"π GPU Memory Cached: {torch.cuda.memory_reserved() // 1024**2} MB")
|
| 125 |
|
|
|
|
| 130 |
base_model = "Qwen/Qwen2.5-0.5B-Instruct"
|
| 131 |
self.tokenizer = AutoTokenizer.from_pretrained(base_model)
|
| 132 |
|
| 133 |
+
# Fix pad token for fallback too
|
| 134 |
+
if self.tokenizer.pad_token is None:
|
| 135 |
+
self.tokenizer.pad_token = self.tokenizer.unk_token or "[PAD]"
|
| 136 |
+
|
| 137 |
if self.device == "cuda":
|
| 138 |
self.model = AutoModelForCausalLM.from_pretrained(
|
| 139 |
base_model,
|
| 140 |
torch_dtype=torch.float16,
|
| 141 |
device_map="auto"
|
| 142 |
)
|
| 143 |
+
if self.model.device.type != "cuda":
|
| 144 |
+
self.model = self.model.to(self.device)
|
| 145 |
else:
|
| 146 |
self.model = AutoModelForCausalLM.from_pretrained(
|
| 147 |
base_model,
|
|
|
|
| 152 |
|
| 153 |
self.model.eval()
|
| 154 |
self.model_loaded = True
|
| 155 |
+
model_device = next(self.model.parameters()).device
|
| 156 |
+
print(f"β
Fallback model loaded on {model_device}")
|
| 157 |
except Exception as fallback_error:
|
| 158 |
print(f"β Fallback also failed: {fallback_error}")
|
| 159 |
self.loading = False
|
| 160 |
|
| 161 |
@spaces.GPU if SPACES_AVAILABLE else lambda func: func
|
| 162 |
def generate_response_gpu(self, conversation: str) -> str:
|
| 163 |
+
"""Generate response using the loaded model with proper device handling"""
|
| 164 |
|
| 165 |
if not self.model_loaded:
|
| 166 |
return "β Model not loaded"
|
| 167 |
|
| 168 |
try:
|
| 169 |
+
# Ensure model is on the correct device
|
| 170 |
+
if self.device == "cuda" and self.model.device.type != "cuda":
|
| 171 |
+
print(f"π Moving model from {self.model.device} to {self.device}")
|
| 172 |
+
self.model = self.model.to(self.device)
|
| 173 |
|
| 174 |
+
# Tokenize input with attention mask
|
| 175 |
+
inputs = self.tokenizer(
|
| 176 |
+
conversation,
|
| 177 |
+
return_tensors="pt",
|
| 178 |
+
padding=True,
|
| 179 |
+
truncation=True,
|
| 180 |
+
max_length=2048,
|
| 181 |
+
add_special_tokens=True
|
| 182 |
+
)
|
| 183 |
+
|
| 184 |
+
# Move inputs to same device as model
|
| 185 |
+
input_ids = inputs['input_ids'].to(self.device)
|
| 186 |
+
attention_mask = inputs['attention_mask'].to(self.device)
|
| 187 |
+
|
| 188 |
+
print(f"π Model device: {self.model.device}, Input device: {input_ids.device}")
|
| 189 |
|
| 190 |
+
# Generate response with proper attention mask
|
| 191 |
with torch.no_grad():
|
| 192 |
outputs = self.model.generate(
|
| 193 |
+
input_ids=input_ids,
|
| 194 |
+
attention_mask=attention_mask, # Pass attention mask
|
| 195 |
max_new_tokens=200,
|
| 196 |
do_sample=True,
|
| 197 |
temperature=0.9,
|
|
|
|
| 206 |
|
| 207 |
# Decode response
|
| 208 |
full_response = self.tokenizer.decode(outputs[0], skip_special_tokens=True)
|
| 209 |
+
response = full_response[len(self.tokenizer.decode(input_ids[0], skip_special_tokens=True)):].strip()
|
| 210 |
|
| 211 |
return self._clean_response(response)
|
| 212 |
|
| 213 |
except Exception as e:
|
| 214 |
+
print(f"β Generation error: {e}")
|
| 215 |
return f"πΈ *Creed scratches his head* Something weird happened... {str(e)[:100]}"
|
| 216 |
|
| 217 |
def generate_response(self, message: str, history: List[Dict]) -> Iterator[str]:
|
|
|
|
| 389 |
|
| 390 |
# Memory status if GPU available
|
| 391 |
if torch.cuda.is_available() and creed_ai.model_loaded:
|
| 392 |
+
print(f"π― Model device verification: {next(creed_ai.model.parameters()).device}")
|
| 393 |
print(f"π₯ Final GPU Memory: {torch.cuda.memory_allocated() // 1024**2} MB allocated")
|
| 394 |
print(f"π GPU Memory Reserved: {torch.cuda.memory_reserved() // 1024**2} MB reserved")
|
| 395 |
|
|
|
|
| 829 |
</div>
|
| 830 |
""")
|
| 831 |
|
| 832 |
+
# Launch with modern styling and public sharing
|
| 833 |
print("π Launching Real Creed AI with modern glassmorphism design...")
|
| 834 |
|
| 835 |
demo.launch(
|
| 836 |
ssr_mode=False,
|
| 837 |
server_name="0.0.0.0",
|
| 838 |
server_port=7860,
|
| 839 |
+
share=True, # Create public link
|
| 840 |
show_error=True
|
| 841 |
)
|
| 842 |
|