Nafiul Huda commited on
Update app.py
Browse files
app.py
CHANGED
|
@@ -99,22 +99,22 @@ model = None
|
|
| 99 |
llm_pipeline = None
|
| 100 |
model_lock = threading.Lock()
|
| 101 |
|
|
|
|
| 102 |
def load_model_on_demand():
|
| 103 |
global tokenizer, model, llm_pipeline
|
| 104 |
with model_lock:
|
| 105 |
if llm_pipeline is None:
|
| 106 |
-
print(f"
|
| 107 |
-
device = "cuda" if torch.cuda.is_available() else "cpu"
|
| 108 |
tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
|
| 109 |
model = AutoModelForCausalLM.from_pretrained(
|
| 110 |
MODEL_NAME,
|
| 111 |
-
torch_dtype=torch.
|
| 112 |
-
device_map=
|
| 113 |
)
|
| 114 |
-
llm_pipeline = pipeline("text-generation", model=model, tokenizer=tokenizer)
|
| 115 |
-
print("Model load complete.")
|
| 116 |
|
| 117 |
-
# Trigger loading of model in background
|
| 118 |
threading.Thread(target=load_model_on_demand, daemon=True).start()
|
| 119 |
|
| 120 |
async def generate_speech(text, rate, pitch):
|
|
@@ -127,8 +127,22 @@ async def generate_speech(text, rate, pitch):
|
|
| 127 |
@spaces.GPU
|
| 128 |
def generate_llm_response(prompt):
|
| 129 |
load_model_on_demand()
|
| 130 |
-
|
| 131 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 132 |
|
| 133 |
# Async Gradio chat handler
|
| 134 |
async def chat_func(user_input, history):
|
|
@@ -143,7 +157,7 @@ async def chat_func(user_input, history):
|
|
| 143 |
|
| 144 |
prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
|
| 145 |
|
| 146 |
-
#
|
| 147 |
response_text = generate_llm_response(prompt)
|
| 148 |
|
| 149 |
# Clean up JSON formatting
|
|
@@ -259,6 +273,27 @@ body { margin: 0; padding: 0; background-color: #ffe082 !important; overflow: hi
|
|
| 259 |
.sdot:nth-child(3) { animation-delay: 0.3s; }
|
| 260 |
@keyframes sdotBounce { from { transform: scaleY(1); } to { transform: scaleY(2.2); } }
|
| 261 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 262 |
/* Hide default Gradio layout elements */
|
| 263 |
footer { display: none !important; }
|
| 264 |
#component-1, #component-2 { background: transparent !important; }
|
|
@@ -481,10 +516,16 @@ function setupApp(container) {{
|
|
| 481 |
recognition.onstart = () => {{
|
| 482 |
micBtn.style.background = 'rgba(255, 75, 75, 0.9)';
|
| 483 |
micBtn.style.color = '#fff';
|
|
|
|
|
|
|
|
|
|
| 484 |
}};
|
| 485 |
recognition.onend = () => {{
|
| 486 |
micBtn.style.background = 'rgba(15, 18, 32, 0.92)';
|
| 487 |
micBtn.style.color = '#6cf';
|
|
|
|
|
|
|
|
|
|
| 488 |
}};
|
| 489 |
recognition.onresult = (event) => {{
|
| 490 |
const transcript = event.results[0][0].transcript;
|
|
@@ -522,6 +563,7 @@ window.addEventListener('load', () => {{
|
|
| 522 |
</script>
|
| 523 |
"""
|
| 524 |
|
|
|
|
| 525 |
model_warning = ""
|
| 526 |
if not os.path.exists(VRM_MODEL_PATH):
|
| 527 |
model_warning = """
|
|
@@ -530,7 +572,7 @@ if not os.path.exists(VRM_MODEL_PATH):
|
|
| 530 |
box-shadow: 0 4px 20px rgba(0,0,0,0.4); font-family: sans-serif; text-align: center; max-width: 90%;
|
| 531 |
border: 2px dashed #ffcccc;">
|
| 532 |
<strong style="font-size: 1.1em;">⚠️ 3D Avatar File Missing!</strong><br>
|
| 533 |
-
Please upload your 3D avatar file and save it
|
| 534 |
</div>
|
| 535 |
"""
|
| 536 |
|
|
@@ -544,6 +586,7 @@ with gr.Blocks() as demo:
|
|
| 544 |
gr.HTML("""
|
| 545 |
<div id="c"></div>
|
| 546 |
<div id="speakDot"><div class="sdot"></div><div class="sdot"></div><div class="sdot"></div></div>
|
|
|
|
| 547 |
<button id="micBtn" style="position: fixed; left: 15px; bottom: calc(15px + env(safe-area-inset-bottom,0)); width: 48px; height: 48px; border-radius: 50%; border: none; font-size: 18px; cursor: pointer; display: flex; align-items: center; justify-content: center; background: rgba(15, 18, 32, 0.92); color: #6cf; border: 1px solid rgba(108, 204, 255, 0.3); backdrop-filter: blur(12px); z-index: 1001; transition: all 0.12s;">🎙️</button>
|
| 548 |
""")
|
| 549 |
|
|
@@ -566,6 +609,9 @@ with gr.Blocks() as demo:
|
|
| 566 |
|
| 567 |
update_js = """
|
| 568 |
(text, hist, audio_path, data) => {
|
|
|
|
|
|
|
|
|
|
| 569 |
let audio_url = "";
|
| 570 |
if (audio_path) {
|
| 571 |
if (typeof audio_path === 'string') {
|
|
@@ -587,10 +633,20 @@ with gr.Blocks() as demo:
|
|
| 587 |
}
|
| 588 |
"""
|
| 589 |
|
| 590 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 591 |
None, [text_input, history, audio_out, data_out], js=update_js
|
| 592 |
)
|
| 593 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 594 |
None, [text_input, history, audio_out, data_out], js=update_js
|
| 595 |
)
|
| 596 |
|
|
|
|
| 99 |
llm_pipeline = None
|
| 100 |
model_lock = threading.Lock()
|
| 101 |
|
| 102 |
+
# Load Model strictly to CPU on startup to respect ZeroGPU design
|
| 103 |
def load_model_on_demand():
|
| 104 |
global tokenizer, model, llm_pipeline
|
| 105 |
with model_lock:
|
| 106 |
if llm_pipeline is None:
|
| 107 |
+
print(f"Loading local LLM onto CPU: {MODEL_NAME}...")
|
|
|
|
| 108 |
tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
|
| 109 |
model = AutoModelForCausalLM.from_pretrained(
|
| 110 |
MODEL_NAME,
|
| 111 |
+
torch_dtype=torch.bfloat16,
|
| 112 |
+
device_map=None
|
| 113 |
)
|
| 114 |
+
llm_pipeline = pipeline("text-generation", model=model, tokenizer=tokenizer, device="cpu")
|
| 115 |
+
print("Model load on CPU complete.")
|
| 116 |
|
| 117 |
+
# Trigger loading of model in background thread
|
| 118 |
threading.Thread(target=load_model_on_demand, daemon=True).start()
|
| 119 |
|
| 120 |
async def generate_speech(text, rate, pitch):
|
|
|
|
| 127 |
@spaces.GPU
|
| 128 |
def generate_llm_response(prompt):
|
| 129 |
load_model_on_demand()
|
| 130 |
+
|
| 131 |
+
# Safely move model to GPU strictly inside active GPU context
|
| 132 |
+
model.to("cuda")
|
| 133 |
+
|
| 134 |
+
inputs = tokenizer(prompt, return_tensors="pt").to("cuda")
|
| 135 |
+
with torch.no_grad():
|
| 136 |
+
outputs = model.generate(
|
| 137 |
+
**inputs,
|
| 138 |
+
max_new_tokens=256,
|
| 139 |
+
do_sample=True,
|
| 140 |
+
temperature=0.75,
|
| 141 |
+
pad_token_id=tokenizer.eos_token_id
|
| 142 |
+
)
|
| 143 |
+
|
| 144 |
+
response = tokenizer.decode(outputs[0][inputs.input_ids.shape[1]:], skip_special_tokens=True)
|
| 145 |
+
return response.strip()
|
| 146 |
|
| 147 |
# Async Gradio chat handler
|
| 148 |
async def chat_func(user_input, history):
|
|
|
|
| 157 |
|
| 158 |
prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
|
| 159 |
|
| 160 |
+
# Run pipeline on the ZeroGPU helper
|
| 161 |
response_text = generate_llm_response(prompt)
|
| 162 |
|
| 163 |
# Clean up JSON formatting
|
|
|
|
| 273 |
.sdot:nth-child(3) { animation-delay: 0.3s; }
|
| 274 |
@keyframes sdotBounce { from { transform: scaleY(1); } to { transform: scaleY(2.2); } }
|
| 275 |
|
| 276 |
+
/* Loading overlay design */
|
| 277 |
+
#load {
|
| 278 |
+
position: fixed;
|
| 279 |
+
top: env(safe-area-inset-top, 15px);
|
| 280 |
+
left: 50%;
|
| 281 |
+
transform: translateX(-50%);
|
| 282 |
+
background: rgba(15, 18, 32, 0.92);
|
| 283 |
+
padding: 8px 20px;
|
| 284 |
+
border-radius: 14px;
|
| 285 |
+
font-size: 12px;
|
| 286 |
+
color: #6cf;
|
| 287 |
+
z-index: 2000;
|
| 288 |
+
backdrop-filter: blur(8px);
|
| 289 |
+
opacity: 0;
|
| 290 |
+
transition: opacity 0.2s;
|
| 291 |
+
pointer-events: none;
|
| 292 |
+
white-space: nowrap;
|
| 293 |
+
border: 1px solid rgba(108, 204, 255, 0.2);
|
| 294 |
+
}
|
| 295 |
+
#load.on { opacity: 1; }
|
| 296 |
+
|
| 297 |
/* Hide default Gradio layout elements */
|
| 298 |
footer { display: none !important; }
|
| 299 |
#component-1, #component-2 { background: transparent !important; }
|
|
|
|
| 516 |
recognition.onstart = () => {{
|
| 517 |
micBtn.style.background = 'rgba(255, 75, 75, 0.9)';
|
| 518 |
micBtn.style.color = '#fff';
|
| 519 |
+
const loadOverlay = document.getElementById('load');
|
| 520 |
+
loadOverlay.textContent = "Listening...";
|
| 521 |
+
loadOverlay.classList.add('on');
|
| 522 |
}};
|
| 523 |
recognition.onend = () => {{
|
| 524 |
micBtn.style.background = 'rgba(15, 18, 32, 0.92)';
|
| 525 |
micBtn.style.color = '#6cf';
|
| 526 |
+
const loadOverlay = document.getElementById('load');
|
| 527 |
+
loadOverlay.classList.remove('on');
|
| 528 |
+
loadOverlay.textContent = "thinking…";
|
| 529 |
}};
|
| 530 |
recognition.onresult = (event) => {{
|
| 531 |
const transcript = event.results[0][0].transcript;
|
|
|
|
| 563 |
</script>
|
| 564 |
"""
|
| 565 |
|
| 566 |
+
# Construct a warning system in Gradio if the model file hasn't been uploaded yet
|
| 567 |
model_warning = ""
|
| 568 |
if not os.path.exists(VRM_MODEL_PATH):
|
| 569 |
model_warning = """
|
|
|
|
| 572 |
box-shadow: 0 4px 20px rgba(0,0,0,0.4); font-family: sans-serif; text-align: center; max-width: 90%;
|
| 573 |
border: 2px dashed #ffcccc;">
|
| 574 |
<strong style="font-size: 1.1em;">⚠️ 3D Avatar File Missing!</strong><br>
|
| 575 |
+
Please upload your 3D avatar file to your Space and save it at: <code>model/Ani.vrm</code>.
|
| 576 |
</div>
|
| 577 |
"""
|
| 578 |
|
|
|
|
| 586 |
gr.HTML("""
|
| 587 |
<div id="c"></div>
|
| 588 |
<div id="speakDot"><div class="sdot"></div><div class="sdot"></div><div class="sdot"></div></div>
|
| 589 |
+
<div id="load">thinking…</div>
|
| 590 |
<button id="micBtn" style="position: fixed; left: 15px; bottom: calc(15px + env(safe-area-inset-bottom,0)); width: 48px; height: 48px; border-radius: 50%; border: none; font-size: 18px; cursor: pointer; display: flex; align-items: center; justify-content: center; background: rgba(15, 18, 32, 0.92); color: #6cf; border: 1px solid rgba(108, 204, 255, 0.3); backdrop-filter: blur(12px); z-index: 1001; transition: all 0.12s;">🎙️</button>
|
| 591 |
""")
|
| 592 |
|
|
|
|
| 609 |
|
| 610 |
update_js = """
|
| 611 |
(text, hist, audio_path, data) => {
|
| 612 |
+
// Deactivate loading indicator instantly upon receiving response
|
| 613 |
+
document.getElementById('load').classList.remove('on');
|
| 614 |
+
|
| 615 |
let audio_url = "";
|
| 616 |
if (audio_path) {
|
| 617 |
if (typeof audio_path === 'string') {
|
|
|
|
| 633 |
}
|
| 634 |
"""
|
| 635 |
|
| 636 |
+
# Chain loaders dynamically: activate overlay instantly on client submit, then run server callbacks
|
| 637 |
+
text_input.submit(
|
| 638 |
+
None, None, None, js="() => { document.getElementById('load').classList.add('on'); }"
|
| 639 |
+
).then(
|
| 640 |
+
on_submit, [text_input, history], [text_input, history, audio_out, data_out]
|
| 641 |
+
).then(
|
| 642 |
None, [text_input, history, audio_out, data_out], js=update_js
|
| 643 |
)
|
| 644 |
+
|
| 645 |
+
send_btn.click(
|
| 646 |
+
None, None, None, js="() => { document.getElementById('load').classList.add('on'); }"
|
| 647 |
+
).then(
|
| 648 |
+
on_submit, [text_input, history], [text_input, history, audio_out, data_out]
|
| 649 |
+
).then(
|
| 650 |
None, [text_input, history, audio_out, data_out], js=update_js
|
| 651 |
)
|
| 652 |
|