Nafiul Huda commited on
Update app.py
Browse files
app.py
CHANGED
|
@@ -28,29 +28,24 @@ ANIMATION_ZIP_PATH = "animation/all_vrma.zip"
|
|
| 28 |
os.makedirs("model", exist_ok=True)
|
| 29 |
os.makedirs("animation", exist_ok=True)
|
| 30 |
|
| 31 |
-
#
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
)
|
| 40 |
-
# Use a fast pipeline
|
| 41 |
-
llm_pipeline = pipeline("text-generation", model=model, tokenizer=tokenizer)
|
| 42 |
|
| 43 |
-
# Helper: Get list of animations from
|
| 44 |
def get_animation_list():
|
| 45 |
-
if not os.path.exists(ANIMATION_ZIP_PATH):
|
| 46 |
-
# Default mock list if file doesn't exist yet
|
| 47 |
-
return ["neutral2.vrma", "neutral3.vrma", "happy.vrma", "wave.vrma", "dance.vrma"]
|
| 48 |
try:
|
| 49 |
-
|
| 50 |
-
|
|
|
|
| 51 |
except Exception as e:
|
| 52 |
print(f"Error reading animations: {e}")
|
| 53 |
-
|
| 54 |
|
| 55 |
ANIMATIONS_LIST = get_animation_list()
|
| 56 |
EMOTIONS = [
|
|
@@ -92,19 +87,31 @@ Example:
|
|
| 92 |
"motion": "idle"
|
| 93 |
}}"""
|
| 94 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 95 |
async def generate_speech(text, rate, pitch):
|
| 96 |
communicate = edge_tts.Communicate(text, VOICE, rate=rate, pitch=pitch)
|
| 97 |
filename = f"speech_{uuid.uuid4().hex}.mp3"
|
| 98 |
await communicate.save(filename)
|
| 99 |
return filename
|
| 100 |
|
| 101 |
-
# Synchronous helper
|
| 102 |
@spaces.GPU
|
| 103 |
def generate_llm_response(prompt):
|
| 104 |
outputs = llm_pipeline(prompt, max_new_tokens=256, do_sample=True, temperature=0.75)
|
| 105 |
return outputs[0]["generated_text"][len(prompt):].strip()
|
| 106 |
|
| 107 |
-
# Async chat
|
| 108 |
async def chat_func(user_input, history):
|
| 109 |
if not user_input:
|
| 110 |
return "", history, None, {}
|
|
@@ -118,10 +125,10 @@ async def chat_func(user_input, history):
|
|
| 118 |
|
| 119 |
prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
|
| 120 |
|
| 121 |
-
#
|
| 122 |
response_text = generate_llm_response(prompt)
|
| 123 |
|
| 124 |
-
# Clean up JSON if LLM added markdown
|
| 125 |
json_str = response_text
|
| 126 |
if "```" in json_str:
|
| 127 |
json_str = json_str.split("```")[1]
|
|
@@ -241,10 +248,10 @@ HTML_HEAD = """
|
|
| 241 |
<script type="importmap">
|
| 242 |
{
|
| 243 |
"imports": {
|
| 244 |
-
"three": "https://
|
| 245 |
-
"three/addons/": "https://
|
| 246 |
-
"@pixiv/three-vrm": "https://
|
| 247 |
-
"@pixiv/three-vrm-animation": "https://
|
| 248 |
}
|
| 249 |
}
|
| 250 |
</script>
|
|
@@ -284,24 +291,45 @@ async () => {
|
|
| 284 |
let audioContext, analyser, dataArray;
|
| 285 |
let isSpeaking = false;
|
| 286 |
let blinkT = 0, nextBlink = 3, blinkV = 0;
|
| 287 |
-
|
|
|
|
|
|
|
|
|
|
| 288 |
|
| 289 |
const loader = new GLTFLoader();
|
| 290 |
loader.register(p => new VRMLoaderPlugin(p));
|
| 291 |
loader.register(p => new VRMAnimationLoaderPlugin(p));
|
| 292 |
|
| 293 |
-
//
|
| 294 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 295 |
const vrm = gltf.userData.vrm;
|
| 296 |
VRMUtils.rotateVRM0(vrm);
|
| 297 |
scene.add(vrm.scene);
|
| 298 |
currentVrm = vrm;
|
| 299 |
mixer = new THREE.AnimationMixer(vrm.scene);
|
| 300 |
|
| 301 |
-
// Initial
|
| 302 |
vrm.expressionManager.setValue('neutral', 1.0);
|
| 303 |
|
| 304 |
-
// Auto-reframe
|
| 305 |
const head = vrm.humanoid.getRawBoneNode('head');
|
| 306 |
if (head) {
|
| 307 |
const worldPos = new THREE.Vector3();
|
|
@@ -309,6 +337,9 @@ async () => {
|
|
| 309 |
orbit.target.copy(worldPos);
|
| 310 |
orbit.update();
|
| 311 |
}
|
|
|
|
|
|
|
|
|
|
| 312 |
}, undefined, (e) => console.log("VRM load placeholder or missing. Use model/Ani.vrm"));
|
| 313 |
|
| 314 |
function initAudioSync(audioElement) {
|
|
@@ -332,7 +363,14 @@ async () => {
|
|
| 332 |
if (currentVrm) {
|
| 333 |
if (mixer) mixer.update(dt);
|
| 334 |
|
| 335 |
-
//
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 336 |
if (isSpeaking && analyser) {
|
| 337 |
analyser.getByteFrequencyData(dataArray);
|
| 338 |
let sum = 0;
|
|
@@ -348,8 +386,8 @@ async () => {
|
|
| 348 |
currentVrm.expressionManager.setValue('ee', 0);
|
| 349 |
}
|
| 350 |
|
| 351 |
-
//
|
| 352 |
-
const isEmotionActive =
|
| 353 |
if (!isSpeaking && !isEmotionActive) {
|
| 354 |
blinkT += dt;
|
| 355 |
if (blinkT > nextBlink) {
|
|
@@ -390,16 +428,23 @@ async () => {
|
|
| 390 |
document.getElementById('speakDot').classList.remove('on');
|
| 391 |
};
|
| 392 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 393 |
if (currentVrm && data.expressions) {
|
| 394 |
-
|
| 395 |
['happy','sad','angry','surprised','relaxed','neutral'].forEach(ex => {
|
| 396 |
currentVrm.expressionManager.setValue(ex, 0);
|
| 397 |
});
|
| 398 |
for (const [key, val] of Object.entries(data.expressions)) {
|
| 399 |
-
|
| 400 |
-
|
|
|
|
|
|
|
| 401 |
}
|
| 402 |
-
if (Object.keys(data.expressions).length === 0) currentExpression = 'neutral';
|
| 403 |
}
|
| 404 |
}
|
| 405 |
});
|
|
@@ -412,9 +457,26 @@ async () => {
|
|
| 412 |
}
|
| 413 |
"""
|
| 414 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 415 |
with gr.Blocks() as demo:
|
| 416 |
history = gr.State([])
|
| 417 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 418 |
with gr.Group(elem_id="custom-viewport"):
|
| 419 |
gr.HTML('<div id="c"></div><div id="speakDot"><div class="sdot"></div><div class="sdot"></div><div class="sdot"></div></div>')
|
| 420 |
|
|
@@ -437,12 +499,23 @@ with gr.Blocks() as demo:
|
|
| 437 |
def on_submit(text, hist):
|
| 438 |
return chat_func(text, hist)
|
| 439 |
|
|
|
|
| 440 |
update_js = """
|
| 441 |
(text, hist, audio_path, data) => {
|
|
|
|
| 442 |
if (audio_path) {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 443 |
window.postMessage({
|
| 444 |
type: 'vrm_update',
|
| 445 |
-
audio_url:
|
| 446 |
data: data
|
| 447 |
}, '*');
|
| 448 |
}
|
|
@@ -460,4 +533,11 @@ with gr.Blocks() as demo:
|
|
| 460 |
demo.load(None, js=JS_CODE)
|
| 461 |
|
| 462 |
if __name__ == "__main__":
|
| 463 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 28 |
os.makedirs("model", exist_ok=True)
|
| 29 |
os.makedirs("animation", exist_ok=True)
|
| 30 |
|
| 31 |
+
# Extract animations from zip if available on startup
|
| 32 |
+
if os.path.exists(ANIMATION_ZIP_PATH):
|
| 33 |
+
print("Extracting animations...")
|
| 34 |
+
try:
|
| 35 |
+
with zipfile.ZipFile(ANIMATION_ZIP_PATH, 'r') as z:
|
| 36 |
+
z.extractall("animation")
|
| 37 |
+
except Exception as e:
|
| 38 |
+
print(f"Error extracting animations: {e}")
|
|
|
|
|
|
|
|
|
|
| 39 |
|
| 40 |
+
# Helper: Get list of animations from extracted directory
|
| 41 |
def get_animation_list():
|
|
|
|
|
|
|
|
|
|
| 42 |
try:
|
| 43 |
+
files = [f for f in os.listdir("animation") if f.endswith('.vrma')]
|
| 44 |
+
if files:
|
| 45 |
+
return files
|
| 46 |
except Exception as e:
|
| 47 |
print(f"Error reading animations: {e}")
|
| 48 |
+
return ["neutral2.vrma", "neutral3.vrma", "happy.vrma", "wave.vrma", "dance.vrma"]
|
| 49 |
|
| 50 |
ANIMATIONS_LIST = get_animation_list()
|
| 51 |
EMOTIONS = [
|
|
|
|
| 87 |
"motion": "idle"
|
| 88 |
}}"""
|
| 89 |
|
| 90 |
+
# Load local LLM
|
| 91 |
+
print(f"Loading local LLM: {MODEL_NAME}...")
|
| 92 |
+
device = "cuda" if torch.cuda.is_available() else "cpu"
|
| 93 |
+
tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
|
| 94 |
+
model = AutoModelForCausalLM.from_pretrained(
|
| 95 |
+
MODEL_NAME,
|
| 96 |
+
torch_dtype=torch.float16 if device == "cuda" else torch.float32,
|
| 97 |
+
device_map="auto" if device == "cuda" else None
|
| 98 |
+
)
|
| 99 |
+
# Use a fast pipeline
|
| 100 |
+
llm_pipeline = pipeline("text-generation", model=model, tokenizer=tokenizer)
|
| 101 |
+
|
| 102 |
async def generate_speech(text, rate, pitch):
|
| 103 |
communicate = edge_tts.Communicate(text, VOICE, rate=rate, pitch=pitch)
|
| 104 |
filename = f"speech_{uuid.uuid4().hex}.mp3"
|
| 105 |
await communicate.save(filename)
|
| 106 |
return filename
|
| 107 |
|
| 108 |
+
# Synchronous GPU helper to perform LLM generation
|
| 109 |
@spaces.GPU
|
| 110 |
def generate_llm_response(prompt):
|
| 111 |
outputs = llm_pipeline(prompt, max_new_tokens=256, do_sample=True, temperature=0.75)
|
| 112 |
return outputs[0]["generated_text"][len(prompt):].strip()
|
| 113 |
|
| 114 |
+
# Async chat handler handles I/O and non-GPU tasks
|
| 115 |
async def chat_func(user_input, history):
|
| 116 |
if not user_input:
|
| 117 |
return "", history, None, {}
|
|
|
|
| 125 |
|
| 126 |
prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
|
| 127 |
|
| 128 |
+
# Run pipeline on the ZeroGPU helper
|
| 129 |
response_text = generate_llm_response(prompt)
|
| 130 |
|
| 131 |
+
# Clean up JSON if LLM added markdown wrappers
|
| 132 |
json_str = response_text
|
| 133 |
if "```" in json_str:
|
| 134 |
json_str = json_str.split("```")[1]
|
|
|
|
| 248 |
<script type="importmap">
|
| 249 |
{
|
| 250 |
"imports": {
|
| 251 |
+
"three": "https://cdn.jsdelivr.net/npm/three@0.160.0/build/three.module.js",
|
| 252 |
+
"three/addons/": "https://cdn.jsdelivr.net/npm/three@0.160.0/examples/jsm/",
|
| 253 |
+
"@pixiv/three-vrm": "https://cdn.jsdelivr.net/npm/@pixiv/three-vrm@3.2.0/lib/three-vrm.module.min.js",
|
| 254 |
+
"@pixiv/three-vrm-animation": "https://cdn.jsdelivr.net/npm/@pixiv/three-vrm-animation@3.2.0/lib/three-vrm-animation.module.min.js"
|
| 255 |
}
|
| 256 |
}
|
| 257 |
</script>
|
|
|
|
| 291 |
let audioContext, analyser, dataArray;
|
| 292 |
let isSpeaking = false;
|
| 293 |
let blinkT = 0, nextBlink = 3, blinkV = 0;
|
| 294 |
+
|
| 295 |
+
// Store current expressions and state so the mixer doesn't reset them on update
|
| 296 |
+
let activeExpressions = { 'neutral': 1.0 };
|
| 297 |
+
let currentVrmaAction = null;
|
| 298 |
|
| 299 |
const loader = new GLTFLoader();
|
| 300 |
loader.register(p => new VRMLoaderPlugin(p));
|
| 301 |
loader.register(p => new VRMAnimationLoaderPlugin(p));
|
| 302 |
|
| 303 |
+
// Dynamic VRMA Animation Player
|
| 304 |
+
function playVrma(vrmaName) {
|
| 305 |
+
if (!currentVrm || !mixer) return;
|
| 306 |
+
const vrmaUrl = `/gradio_api/file=animation/${vrmaName}`;
|
| 307 |
+
loader.load(vrmaUrl, (gltf) => {
|
| 308 |
+
const vrmAnimations = gltf.userData.vrmAnimations;
|
| 309 |
+
if (vrmAnimations && vrmAnimations.length > 0) {
|
| 310 |
+
if (currentVrmaAction) {
|
| 311 |
+
currentVrmaAction.stop();
|
| 312 |
+
mixer.uncacheClip(currentVrmaAction.getClip());
|
| 313 |
+
}
|
| 314 |
+
const clip = createVRMAnimationClip(vrmAnimations[0], currentVrm);
|
| 315 |
+
currentVrmaAction = mixer.clipAction(clip);
|
| 316 |
+
currentVrmaAction.play();
|
| 317 |
+
}
|
| 318 |
+
}, undefined, (err) => console.log("VRMA loading skipped or file missing:", vrmaName));
|
| 319 |
+
}
|
| 320 |
+
|
| 321 |
+
// Load VRM Model securely using /gradio_api/file=
|
| 322 |
+
loader.load('/gradio_api/file=model/Ani.vrm', (gltf) => {
|
| 323 |
const vrm = gltf.userData.vrm;
|
| 324 |
VRMUtils.rotateVRM0(vrm);
|
| 325 |
scene.add(vrm.scene);
|
| 326 |
currentVrm = vrm;
|
| 327 |
mixer = new THREE.AnimationMixer(vrm.scene);
|
| 328 |
|
| 329 |
+
// Initial Pose Setup
|
| 330 |
vrm.expressionManager.setValue('neutral', 1.0);
|
| 331 |
|
| 332 |
+
// Auto-reframe on face
|
| 333 |
const head = vrm.humanoid.getRawBoneNode('head');
|
| 334 |
if (head) {
|
| 335 |
const worldPos = new THREE.Vector3();
|
|
|
|
| 337 |
orbit.target.copy(worldPos);
|
| 338 |
orbit.update();
|
| 339 |
}
|
| 340 |
+
|
| 341 |
+
// Start default/idle animation
|
| 342 |
+
playVrma('neutral2.vrma');
|
| 343 |
}, undefined, (e) => console.log("VRM load placeholder or missing. Use model/Ani.vrm"));
|
| 344 |
|
| 345 |
function initAudioSync(audioElement) {
|
|
|
|
| 363 |
if (currentVrm) {
|
| 364 |
if (mixer) mixer.update(dt);
|
| 365 |
|
| 366 |
+
// Reapply facial expressions on every frame after mixer update to prevent resets
|
| 367 |
+
if (activeExpressions) {
|
| 368 |
+
for (const [key, val] of Object.entries(activeExpressions)) {
|
| 369 |
+
currentVrm.expressionManager.setValue(key, val);
|
| 370 |
+
}
|
| 371 |
+
}
|
| 372 |
+
|
| 373 |
+
// Natural dynamic Lip Sync
|
| 374 |
if (isSpeaking && analyser) {
|
| 375 |
analyser.getByteFrequencyData(dataArray);
|
| 376 |
let sum = 0;
|
|
|
|
| 386 |
currentVrm.expressionManager.setValue('ee', 0);
|
| 387 |
}
|
| 388 |
|
| 389 |
+
// Natural Eye Blinking
|
| 390 |
+
const isEmotionActive = activeExpressions && (activeExpressions.happy > 0.5 || activeExpressions.surprised > 0.5 || activeExpressions.relaxed > 0.5);
|
| 391 |
if (!isSpeaking && !isEmotionActive) {
|
| 392 |
blinkT += dt;
|
| 393 |
if (blinkT > nextBlink) {
|
|
|
|
| 428 |
document.getElementById('speakDot').classList.remove('on');
|
| 429 |
};
|
| 430 |
|
| 431 |
+
// Process dynamic animations returned from LLM
|
| 432 |
+
if (data.vrma) {
|
| 433 |
+
playVrma(data.vrma);
|
| 434 |
+
}
|
| 435 |
+
|
| 436 |
+
// Process active facial expression changes
|
| 437 |
if (currentVrm && data.expressions) {
|
| 438 |
+
activeExpressions = {};
|
| 439 |
['happy','sad','angry','surprised','relaxed','neutral'].forEach(ex => {
|
| 440 |
currentVrm.expressionManager.setValue(ex, 0);
|
| 441 |
});
|
| 442 |
for (const [key, val] of Object.entries(data.expressions)) {
|
| 443 |
+
activeExpressions[key] = val;
|
| 444 |
+
}
|
| 445 |
+
if (Object.keys(activeExpressions).length === 0) {
|
| 446 |
+
activeExpressions['neutral'] = 1.0;
|
| 447 |
}
|
|
|
|
| 448 |
}
|
| 449 |
}
|
| 450 |
});
|
|
|
|
| 457 |
}
|
| 458 |
"""
|
| 459 |
|
| 460 |
+
# Construct a warning system in Gradio if the model file hasn't been uploaded yet
|
| 461 |
+
model_warning = ""
|
| 462 |
+
if not os.path.exists(VRM_MODEL_PATH):
|
| 463 |
+
model_warning = """
|
| 464 |
+
<div style="position: fixed; top: 15px; left: 50%; transform: translateX(-50%); z-index: 9999;
|
| 465 |
+
background: rgba(255, 75, 75, 0.95); color: white; padding: 15px 30px; border-radius: 12px;
|
| 466 |
+
box-shadow: 0 4px 20px rgba(0,0,0,0.4); font-family: sans-serif; text-align: center; max-width: 90%;
|
| 467 |
+
border: 2px dashed #ffcccc;">
|
| 468 |
+
<strong style="font-size: 1.1em;">⚠️ 3D Avatar File Missing!</strong><br>
|
| 469 |
+
Please upload your 3D avatar file to your Space and save it at: <code>model/Ani.vrm</code>.
|
| 470 |
+
</div>
|
| 471 |
+
"""
|
| 472 |
+
|
| 473 |
with gr.Blocks() as demo:
|
| 474 |
history = gr.State([])
|
| 475 |
|
| 476 |
+
# Display the file placeholder overlay if necessary
|
| 477 |
+
if model_warning:
|
| 478 |
+
gr.HTML(model_warning)
|
| 479 |
+
|
| 480 |
with gr.Group(elem_id="custom-viewport"):
|
| 481 |
gr.HTML('<div id="c"></div><div id="speakDot"><div class="sdot"></div><div class="sdot"></div><div class="sdot"></div></div>')
|
| 482 |
|
|
|
|
| 499 |
def on_submit(text, hist):
|
| 500 |
return chat_func(text, hist)
|
| 501 |
|
| 502 |
+
# Clean Javascript parser supporting both old and new Gradio structures for passing file-paths
|
| 503 |
update_js = """
|
| 504 |
(text, hist, audio_path, data) => {
|
| 505 |
+
let audio_url = "";
|
| 506 |
if (audio_path) {
|
| 507 |
+
if (typeof audio_path === 'string') {
|
| 508 |
+
audio_url = audio_path;
|
| 509 |
+
} else if (audio_path.url) {
|
| 510 |
+
audio_url = audio_path.url;
|
| 511 |
+
} else if (audio_path.path) {
|
| 512 |
+
audio_url = "/gradio_api/file=" + audio_path.path;
|
| 513 |
+
}
|
| 514 |
+
}
|
| 515 |
+
if (audio_url && data) {
|
| 516 |
window.postMessage({
|
| 517 |
type: 'vrm_update',
|
| 518 |
+
audio_url: audio_url,
|
| 519 |
data: data
|
| 520 |
}, '*');
|
| 521 |
}
|
|
|
|
| 533 |
demo.load(None, js=JS_CODE)
|
| 534 |
|
| 535 |
if __name__ == "__main__":
|
| 536 |
+
# Declare "allowed_paths" with working directory to allow local file serving safely
|
| 537 |
+
demo.launch(
|
| 538 |
+
server_name="0.0.0.0",
|
| 539 |
+
server_port=7860,
|
| 540 |
+
css=CSS,
|
| 541 |
+
head=HTML_HEAD,
|
| 542 |
+
allowed_paths=["."]
|
| 543 |
+
)
|