Nafiul Huda commited on
Update app.py
Browse files
app.py
CHANGED
|
@@ -8,7 +8,6 @@ import gradio as gr
|
|
| 8 |
import edge_tts
|
| 9 |
from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
|
| 10 |
import uuid
|
| 11 |
-
import threading
|
| 12 |
|
| 13 |
# Fallback wrapper for ZeroGPU compatibility
|
| 14 |
try:
|
|
@@ -93,29 +92,16 @@ Example:
|
|
| 93 |
"motion": "idle"
|
| 94 |
}}"""
|
| 95 |
|
| 96 |
-
#
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
if llm_pipeline is None:
|
| 107 |
-
print(f"Loading local LLM onto CPU: {MODEL_NAME}...")
|
| 108 |
-
tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
|
| 109 |
-
model = AutoModelForCausalLM.from_pretrained(
|
| 110 |
-
MODEL_NAME,
|
| 111 |
-
torch_dtype=torch.bfloat16,
|
| 112 |
-
device_map=None
|
| 113 |
-
)
|
| 114 |
-
llm_pipeline = pipeline("text-generation", model=model, tokenizer=tokenizer, device="cpu")
|
| 115 |
-
print("Model load on CPU complete.")
|
| 116 |
-
|
| 117 |
-
# Trigger loading of model in background thread
|
| 118 |
-
threading.Thread(target=load_model_on_demand, daemon=True).start()
|
| 119 |
|
| 120 |
async def generate_speech(text, rate, pitch):
|
| 121 |
communicate = edge_tts.Communicate(text, VOICE, rate=rate, pitch=pitch)
|
|
@@ -126,11 +112,7 @@ async def generate_speech(text, rate, pitch):
|
|
| 126 |
# Synchronous GPU helper to perform LLM generation
|
| 127 |
@spaces.GPU
|
| 128 |
def generate_llm_response(prompt):
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
# Safely move model to GPU strictly inside active GPU context
|
| 132 |
-
model.to("cuda")
|
| 133 |
-
|
| 134 |
inputs = tokenizer(prompt, return_tensors="pt").to("cuda")
|
| 135 |
with torch.no_grad():
|
| 136 |
outputs = model.generate(
|
|
|
|
| 8 |
import edge_tts
|
| 9 |
from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
|
| 10 |
import uuid
|
|
|
|
| 11 |
|
| 12 |
# Fallback wrapper for ZeroGPU compatibility
|
| 13 |
try:
|
|
|
|
| 92 |
"motion": "idle"
|
| 93 |
}}"""
|
| 94 |
|
| 95 |
+
# Synchronous module-level loading on the main thread (safely handled by ZeroGPU's emulation)
|
| 96 |
+
print(f"Loading local LLM: {MODEL_NAME}...")
|
| 97 |
+
device = "cuda" if torch.cuda.is_available() else "cpu"
|
| 98 |
+
tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
|
| 99 |
+
model = AutoModelForCausalLM.from_pretrained(
|
| 100 |
+
MODEL_NAME,
|
| 101 |
+
torch_dtype=torch.float16 if device == "cuda" else torch.float32,
|
| 102 |
+
device_map="auto" if device == "cuda" else None
|
| 103 |
+
)
|
| 104 |
+
print("Model loaded successfully.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 105 |
|
| 106 |
async def generate_speech(text, rate, pitch):
|
| 107 |
communicate = edge_tts.Communicate(text, VOICE, rate=rate, pitch=pitch)
|
|
|
|
| 112 |
# Synchronous GPU helper to perform LLM generation
|
| 113 |
@spaces.GPU
|
| 114 |
def generate_llm_response(prompt):
|
| 115 |
+
# Inside the spaces.GPU context, the model is hot-swapped to GPU dynamically
|
|
|
|
|
|
|
|
|
|
|
|
|
| 116 |
inputs = tokenizer(prompt, return_tensors="pt").to("cuda")
|
| 117 |
with torch.no_grad():
|
| 118 |
outputs = model.generate(
|