Nafiul Huda commited on
Commit
d019426
·
verified ·
1 Parent(s): acab719

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +69 -13
app.py CHANGED
@@ -99,22 +99,22 @@ model = None
99
  llm_pipeline = None
100
  model_lock = threading.Lock()
101
 
 
102
  def load_model_on_demand():
103
  global tokenizer, model, llm_pipeline
104
  with model_lock:
105
  if llm_pipeline is None:
106
- print(f"Initializing local LLM on device: {MODEL_NAME}...")
107
- device = "cuda" if torch.cuda.is_available() else "cpu"
108
  tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
109
  model = AutoModelForCausalLM.from_pretrained(
110
  MODEL_NAME,
111
- torch_dtype=torch.float16 if device == "cuda" else torch.float32,
112
- device_map="auto" if device == "cuda" else None
113
  )
114
- llm_pipeline = pipeline("text-generation", model=model, tokenizer=tokenizer)
115
- print("Model load complete.")
116
 
117
- # Trigger loading of model in background to ensure Gradio starts instantly
118
  threading.Thread(target=load_model_on_demand, daemon=True).start()
119
 
120
  async def generate_speech(text, rate, pitch):
@@ -127,8 +127,22 @@ async def generate_speech(text, rate, pitch):
127
  @spaces.GPU
128
  def generate_llm_response(prompt):
129
  load_model_on_demand()
130
- outputs = llm_pipeline(prompt, max_new_tokens=256, do_sample=True, temperature=0.75)
131
- return outputs[0]["generated_text"][len(prompt):].strip()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
132
 
133
  # Async Gradio chat handler
134
  async def chat_func(user_input, history):
@@ -143,7 +157,7 @@ async def chat_func(user_input, history):
143
 
144
  prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
145
 
146
- # Process model prediction
147
  response_text = generate_llm_response(prompt)
148
 
149
  # Clean up JSON formatting
@@ -259,6 +273,27 @@ body { margin: 0; padding: 0; background-color: #ffe082 !important; overflow: hi
259
  .sdot:nth-child(3) { animation-delay: 0.3s; }
260
  @keyframes sdotBounce { from { transform: scaleY(1); } to { transform: scaleY(2.2); } }
261
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
262
  /* Hide default Gradio layout elements */
263
  footer { display: none !important; }
264
  #component-1, #component-2 { background: transparent !important; }
@@ -481,10 +516,16 @@ function setupApp(container) {{
481
  recognition.onstart = () => {{
482
  micBtn.style.background = 'rgba(255, 75, 75, 0.9)';
483
  micBtn.style.color = '#fff';
 
 
 
484
  }};
485
  recognition.onend = () => {{
486
  micBtn.style.background = 'rgba(15, 18, 32, 0.92)';
487
  micBtn.style.color = '#6cf';
 
 
 
488
  }};
489
  recognition.onresult = (event) => {{
490
  const transcript = event.results[0][0].transcript;
@@ -522,6 +563,7 @@ window.addEventListener('load', () => {{
522
  </script>
523
  """
524
 
 
525
  model_warning = ""
526
  if not os.path.exists(VRM_MODEL_PATH):
527
  model_warning = """
@@ -530,7 +572,7 @@ if not os.path.exists(VRM_MODEL_PATH):
530
  box-shadow: 0 4px 20px rgba(0,0,0,0.4); font-family: sans-serif; text-align: center; max-width: 90%;
531
  border: 2px dashed #ffcccc;">
532
  <strong style="font-size: 1.1em;">⚠️ 3D Avatar File Missing!</strong><br>
533
- Please upload your 3D avatar file and save it inside your repository as: <code>model/Ani.vrm</code>.
534
  </div>
535
  """
536
 
@@ -544,6 +586,7 @@ with gr.Blocks() as demo:
544
  gr.HTML("""
545
  <div id="c"></div>
546
  <div id="speakDot"><div class="sdot"></div><div class="sdot"></div><div class="sdot"></div></div>
 
547
  <button id="micBtn" style="position: fixed; left: 15px; bottom: calc(15px + env(safe-area-inset-bottom,0)); width: 48px; height: 48px; border-radius: 50%; border: none; font-size: 18px; cursor: pointer; display: flex; align-items: center; justify-content: center; background: rgba(15, 18, 32, 0.92); color: #6cf; border: 1px solid rgba(108, 204, 255, 0.3); backdrop-filter: blur(12px); z-index: 1001; transition: all 0.12s;">🎙️</button>
548
  """)
549
 
@@ -566,6 +609,9 @@ with gr.Blocks() as demo:
566
 
567
  update_js = """
568
  (text, hist, audio_path, data) => {
 
 
 
569
  let audio_url = "";
570
  if (audio_path) {
571
  if (typeof audio_path === 'string') {
@@ -587,10 +633,20 @@ with gr.Blocks() as demo:
587
  }
588
  """
589
 
590
- text_input.submit(on_submit, [text_input, history], [text_input, history, audio_out, data_out]).then(
 
 
 
 
 
591
  None, [text_input, history, audio_out, data_out], js=update_js
592
  )
593
- send_btn.click(on_submit, [text_input, history], [text_input, history, audio_out, data_out]).then(
 
 
 
 
 
594
  None, [text_input, history, audio_out, data_out], js=update_js
595
  )
596
 
 
99
  llm_pipeline = None
100
  model_lock = threading.Lock()
101
 
102
+ # Load Model strictly to CPU on startup to respect ZeroGPU design
103
  def load_model_on_demand():
104
  global tokenizer, model, llm_pipeline
105
  with model_lock:
106
  if llm_pipeline is None:
107
+ print(f"Loading local LLM onto CPU: {MODEL_NAME}...")
 
108
  tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
109
  model = AutoModelForCausalLM.from_pretrained(
110
  MODEL_NAME,
111
+ torch_dtype=torch.bfloat16,
112
+ device_map=None
113
  )
114
+ llm_pipeline = pipeline("text-generation", model=model, tokenizer=tokenizer, device="cpu")
115
+ print("Model load on CPU complete.")
116
 
117
+ # Trigger loading of model in background thread
118
  threading.Thread(target=load_model_on_demand, daemon=True).start()
119
 
120
  async def generate_speech(text, rate, pitch):
 
127
  @spaces.GPU
128
  def generate_llm_response(prompt):
129
  load_model_on_demand()
130
+
131
+ # Safely move model to GPU strictly inside active GPU context
132
+ model.to("cuda")
133
+
134
+ inputs = tokenizer(prompt, return_tensors="pt").to("cuda")
135
+ with torch.no_grad():
136
+ outputs = model.generate(
137
+ **inputs,
138
+ max_new_tokens=256,
139
+ do_sample=True,
140
+ temperature=0.75,
141
+ pad_token_id=tokenizer.eos_token_id
142
+ )
143
+
144
+ response = tokenizer.decode(outputs[0][inputs.input_ids.shape[1]:], skip_special_tokens=True)
145
+ return response.strip()
146
 
147
  # Async Gradio chat handler
148
  async def chat_func(user_input, history):
 
157
 
158
  prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
159
 
160
+ # Run pipeline on the ZeroGPU helper
161
  response_text = generate_llm_response(prompt)
162
 
163
  # Clean up JSON formatting
 
273
  .sdot:nth-child(3) { animation-delay: 0.3s; }
274
  @keyframes sdotBounce { from { transform: scaleY(1); } to { transform: scaleY(2.2); } }
275
 
276
+ /* Loading overlay design */
277
+ #load {
278
+ position: fixed;
279
+ top: env(safe-area-inset-top, 15px);
280
+ left: 50%;
281
+ transform: translateX(-50%);
282
+ background: rgba(15, 18, 32, 0.92);
283
+ padding: 8px 20px;
284
+ border-radius: 14px;
285
+ font-size: 12px;
286
+ color: #6cf;
287
+ z-index: 2000;
288
+ backdrop-filter: blur(8px);
289
+ opacity: 0;
290
+ transition: opacity 0.2s;
291
+ pointer-events: none;
292
+ white-space: nowrap;
293
+ border: 1px solid rgba(108, 204, 255, 0.2);
294
+ }
295
+ #load.on { opacity: 1; }
296
+
297
  /* Hide default Gradio layout elements */
298
  footer { display: none !important; }
299
  #component-1, #component-2 { background: transparent !important; }
 
516
  recognition.onstart = () => {{
517
  micBtn.style.background = 'rgba(255, 75, 75, 0.9)';
518
  micBtn.style.color = '#fff';
519
+ const loadOverlay = document.getElementById('load');
520
+ loadOverlay.textContent = "Listening...";
521
+ loadOverlay.classList.add('on');
522
  }};
523
  recognition.onend = () => {{
524
  micBtn.style.background = 'rgba(15, 18, 32, 0.92)';
525
  micBtn.style.color = '#6cf';
526
+ const loadOverlay = document.getElementById('load');
527
+ loadOverlay.classList.remove('on');
528
+ loadOverlay.textContent = "thinking…";
529
  }};
530
  recognition.onresult = (event) => {{
531
  const transcript = event.results[0][0].transcript;
 
563
  </script>
564
  """
565
 
566
+ # Construct a warning system in Gradio if the model file hasn't been uploaded yet
567
  model_warning = ""
568
  if not os.path.exists(VRM_MODEL_PATH):
569
  model_warning = """
 
572
  box-shadow: 0 4px 20px rgba(0,0,0,0.4); font-family: sans-serif; text-align: center; max-width: 90%;
573
  border: 2px dashed #ffcccc;">
574
  <strong style="font-size: 1.1em;">⚠️ 3D Avatar File Missing!</strong><br>
575
+ Please upload your 3D avatar file to your Space and save it at: <code>model/Ani.vrm</code>.
576
  </div>
577
  """
578
 
 
586
  gr.HTML("""
587
  <div id="c"></div>
588
  <div id="speakDot"><div class="sdot"></div><div class="sdot"></div><div class="sdot"></div></div>
589
+ <div id="load">thinking…</div>
590
  <button id="micBtn" style="position: fixed; left: 15px; bottom: calc(15px + env(safe-area-inset-bottom,0)); width: 48px; height: 48px; border-radius: 50%; border: none; font-size: 18px; cursor: pointer; display: flex; align-items: center; justify-content: center; background: rgba(15, 18, 32, 0.92); color: #6cf; border: 1px solid rgba(108, 204, 255, 0.3); backdrop-filter: blur(12px); z-index: 1001; transition: all 0.12s;">🎙️</button>
591
  """)
592
 
 
609
 
610
  update_js = """
611
  (text, hist, audio_path, data) => {
612
+ // Deactivate loading indicator instantly upon receiving response
613
+ document.getElementById('load').classList.remove('on');
614
+
615
  let audio_url = "";
616
  if (audio_path) {
617
  if (typeof audio_path === 'string') {
 
633
  }
634
  """
635
 
636
+ # Chain loaders dynamically: activate overlay instantly on client submit, then run server callbacks
637
+ text_input.submit(
638
+ None, None, None, js="() => { document.getElementById('load').classList.add('on'); }"
639
+ ).then(
640
+ on_submit, [text_input, history], [text_input, history, audio_out, data_out]
641
+ ).then(
642
  None, [text_input, history, audio_out, data_out], js=update_js
643
  )
644
+
645
+ send_btn.click(
646
+ None, None, None, js="() => { document.getElementById('load').classList.add('on'); }"
647
+ ).then(
648
+ on_submit, [text_input, history], [text_input, history, audio_out, data_out]
649
+ ).then(
650
  None, [text_input, history, audio_out, data_out], js=update_js
651
  )
652