Nafiul Huda commited on
Commit
6d3a819
·
verified ·
1 Parent(s): d019426

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +11 -29
app.py CHANGED
@@ -8,7 +8,6 @@ import gradio as gr
8
  import edge_tts
9
  from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
10
  import uuid
11
- import threading
12
 
13
  # Fallback wrapper for ZeroGPU compatibility
14
  try:
@@ -93,29 +92,16 @@ Example:
93
  "motion": "idle"
94
  }}"""
95
 
96
- # Placeholders for thread-safe lazy-loaded LLM pipeline
97
- tokenizer = None
98
- model = None
99
- llm_pipeline = None
100
- model_lock = threading.Lock()
101
-
102
- # Load Model strictly to CPU on startup to respect ZeroGPU design
103
- def load_model_on_demand():
104
- global tokenizer, model, llm_pipeline
105
- with model_lock:
106
- if llm_pipeline is None:
107
- print(f"Loading local LLM onto CPU: {MODEL_NAME}...")
108
- tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
109
- model = AutoModelForCausalLM.from_pretrained(
110
- MODEL_NAME,
111
- torch_dtype=torch.bfloat16,
112
- device_map=None
113
- )
114
- llm_pipeline = pipeline("text-generation", model=model, tokenizer=tokenizer, device="cpu")
115
- print("Model load on CPU complete.")
116
-
117
- # Trigger loading of model in background thread
118
- threading.Thread(target=load_model_on_demand, daemon=True).start()
119
 
120
  async def generate_speech(text, rate, pitch):
121
  communicate = edge_tts.Communicate(text, VOICE, rate=rate, pitch=pitch)
@@ -126,11 +112,7 @@ async def generate_speech(text, rate, pitch):
126
  # Synchronous GPU helper to perform LLM generation
127
  @spaces.GPU
128
  def generate_llm_response(prompt):
129
- load_model_on_demand()
130
-
131
- # Safely move model to GPU strictly inside active GPU context
132
- model.to("cuda")
133
-
134
  inputs = tokenizer(prompt, return_tensors="pt").to("cuda")
135
  with torch.no_grad():
136
  outputs = model.generate(
 
8
  import edge_tts
9
  from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
10
  import uuid
 
11
 
12
  # Fallback wrapper for ZeroGPU compatibility
13
  try:
 
92
  "motion": "idle"
93
  }}"""
94
 
95
+ # Synchronous module-level loading on the main thread (safely handled by ZeroGPU's emulation)
96
+ print(f"Loading local LLM: {MODEL_NAME}...")
97
+ device = "cuda" if torch.cuda.is_available() else "cpu"
98
+ tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
99
+ model = AutoModelForCausalLM.from_pretrained(
100
+ MODEL_NAME,
101
+ torch_dtype=torch.float16 if device == "cuda" else torch.float32,
102
+ device_map="auto" if device == "cuda" else None
103
+ )
104
+ print("Model loaded successfully.")
 
 
 
 
 
 
 
 
 
 
 
 
 
105
 
106
  async def generate_speech(text, rate, pitch):
107
  communicate = edge_tts.Communicate(text, VOICE, rate=rate, pitch=pitch)
 
112
  # Synchronous GPU helper to perform LLM generation
113
  @spaces.GPU
114
  def generate_llm_response(prompt):
115
+ # Inside the spaces.GPU context, the model is hot-swapped to GPU dynamically
 
 
 
 
116
  inputs = tokenizer(prompt, return_tensors="pt").to("cuda")
117
  with torch.no_grad():
118
  outputs = model.generate(