Nafiul Huda commited on
Commit
85aa8ce
·
verified ·
1 Parent(s): 7e30780

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +120 -40
app.py CHANGED
@@ -28,29 +28,24 @@ ANIMATION_ZIP_PATH = "animation/all_vrma.zip"
28
  os.makedirs("model", exist_ok=True)
29
  os.makedirs("animation", exist_ok=True)
30
 
31
- # Load local LLM
32
- print(f"Loading local LLM: {MODEL_NAME}...")
33
- device = "cuda" if torch.cuda.is_available() else "cpu"
34
- tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
35
- model = AutoModelForCausalLM.from_pretrained(
36
- MODEL_NAME,
37
- torch_dtype=torch.float16 if device == "cuda" else torch.float32,
38
- device_map="auto" if device == "cuda" else None
39
- )
40
- # Use a fast pipeline
41
- llm_pipeline = pipeline("text-generation", model=model, tokenizer=tokenizer)
42
 
43
- # Helper: Get list of animations from zip
44
  def get_animation_list():
45
- if not os.path.exists(ANIMATION_ZIP_PATH):
46
- # Default mock list if file doesn't exist yet
47
- return ["neutral2.vrma", "neutral3.vrma", "happy.vrma", "wave.vrma", "dance.vrma"]
48
  try:
49
- with zipfile.ZipFile(ANIMATION_ZIP_PATH, 'r') as z:
50
- return [os.path.basename(f) for f in z.namelist() if f.endswith('.vrma')]
 
51
  except Exception as e:
52
  print(f"Error reading animations: {e}")
53
- return ["neutral2.vrma"]
54
 
55
  ANIMATIONS_LIST = get_animation_list()
56
  EMOTIONS = [
@@ -92,19 +87,31 @@ Example:
92
  "motion": "idle"
93
  }}"""
94
 
 
 
 
 
 
 
 
 
 
 
 
 
95
  async def generate_speech(text, rate, pitch):
96
  communicate = edge_tts.Communicate(text, VOICE, rate=rate, pitch=pitch)
97
  filename = f"speech_{uuid.uuid4().hex}.mp3"
98
  await communicate.save(filename)
99
  return filename
100
 
101
- # Synchronous helper decorated with @spaces.GPU for model inference
102
  @spaces.GPU
103
  def generate_llm_response(prompt):
104
  outputs = llm_pipeline(prompt, max_new_tokens=256, do_sample=True, temperature=0.75)
105
  return outputs[0]["generated_text"][len(prompt):].strip()
106
 
107
- # Async chat function handles I/O, history management, and TTS generation
108
  async def chat_func(user_input, history):
109
  if not user_input:
110
  return "", history, None, {}
@@ -118,10 +125,10 @@ async def chat_func(user_input, history):
118
 
119
  prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
120
 
121
- # Call the GPU-decorated synchronous generator
122
  response_text = generate_llm_response(prompt)
123
 
124
- # Clean up JSON if LLM added markdown
125
  json_str = response_text
126
  if "```" in json_str:
127
  json_str = json_str.split("```")[1]
@@ -241,10 +248,10 @@ HTML_HEAD = """
241
  <script type="importmap">
242
  {
243
  "imports": {
244
- "three": "https://esm.sh/three@0.160.0",
245
- "three/addons/": "https://esm.sh/three@0.160.0/examples/jsm/",
246
- "@pixiv/three-vrm": "https://esm.sh/@pixiv/three-vrm@3.3.4?deps=three@0.160.0",
247
- "@pixiv/three-vrm-animation": "https://esm.sh/@pixiv/three-vrm-animation@3.3.4?deps=three@0.160.0,@pixiv/three-vrm@3.3.4"
248
  }
249
  }
250
  </script>
@@ -284,24 +291,45 @@ async () => {
284
  let audioContext, analyser, dataArray;
285
  let isSpeaking = false;
286
  let blinkT = 0, nextBlink = 3, blinkV = 0;
287
- let currentExpression = 'neutral';
 
 
 
288
 
289
  const loader = new GLTFLoader();
290
  loader.register(p => new VRMLoaderPlugin(p));
291
  loader.register(p => new VRMAnimationLoaderPlugin(p));
292
 
293
- // Load VRM Model
294
- loader.load('file/model/Ani.vrm', (gltf) => {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
295
  const vrm = gltf.userData.vrm;
296
  VRMUtils.rotateVRM0(vrm);
297
  scene.add(vrm.scene);
298
  currentVrm = vrm;
299
  mixer = new THREE.AnimationMixer(vrm.scene);
300
 
301
- // Initial Expression
302
  vrm.expressionManager.setValue('neutral', 1.0);
303
 
304
- // Auto-reframe
305
  const head = vrm.humanoid.getRawBoneNode('head');
306
  if (head) {
307
  const worldPos = new THREE.Vector3();
@@ -309,6 +337,9 @@ async () => {
309
  orbit.target.copy(worldPos);
310
  orbit.update();
311
  }
 
 
 
312
  }, undefined, (e) => console.log("VRM load placeholder or missing. Use model/Ani.vrm"));
313
 
314
  function initAudioSync(audioElement) {
@@ -332,7 +363,14 @@ async () => {
332
  if (currentVrm) {
333
  if (mixer) mixer.update(dt);
334
 
335
- // Natural Lip Sync
 
 
 
 
 
 
 
336
  if (isSpeaking && analyser) {
337
  analyser.getByteFrequencyData(dataArray);
338
  let sum = 0;
@@ -348,8 +386,8 @@ async () => {
348
  currentVrm.expressionManager.setValue('ee', 0);
349
  }
350
 
351
- // No Auto-Blinking when speaking or emotions active
352
- const isEmotionActive = currentExpression !== 'neutral';
353
  if (!isSpeaking && !isEmotionActive) {
354
  blinkT += dt;
355
  if (blinkT > nextBlink) {
@@ -390,16 +428,23 @@ async () => {
390
  document.getElementById('speakDot').classList.remove('on');
391
  };
392
 
 
 
 
 
 
 
393
  if (currentVrm && data.expressions) {
394
- // Reset expressions
395
  ['happy','sad','angry','surprised','relaxed','neutral'].forEach(ex => {
396
  currentVrm.expressionManager.setValue(ex, 0);
397
  });
398
  for (const [key, val] of Object.entries(data.expressions)) {
399
- currentVrm.expressionManager.setValue(key, val);
400
- if (val > 0.5) currentExpression = key;
 
 
401
  }
402
- if (Object.keys(data.expressions).length === 0) currentExpression = 'neutral';
403
  }
404
  }
405
  });
@@ -412,9 +457,26 @@ async () => {
412
  }
413
  """
414
 
 
 
 
 
 
 
 
 
 
 
 
 
 
415
  with gr.Blocks() as demo:
416
  history = gr.State([])
417
 
 
 
 
 
418
  with gr.Group(elem_id="custom-viewport"):
419
  gr.HTML('<div id="c"></div><div id="speakDot"><div class="sdot"></div><div class="sdot"></div><div class="sdot"></div></div>')
420
 
@@ -437,12 +499,23 @@ with gr.Blocks() as demo:
437
  def on_submit(text, hist):
438
  return chat_func(text, hist)
439
 
 
440
  update_js = """
441
  (text, hist, audio_path, data) => {
 
442
  if (audio_path) {
 
 
 
 
 
 
 
 
 
443
  window.postMessage({
444
  type: 'vrm_update',
445
- audio_url: audio_path,
446
  data: data
447
  }, '*');
448
  }
@@ -460,4 +533,11 @@ with gr.Blocks() as demo:
460
  demo.load(None, js=JS_CODE)
461
 
462
  if __name__ == "__main__":
463
- demo.launch(server_name="0.0.0.0", server_port=7860, css=CSS, head=HTML_HEAD)
 
 
 
 
 
 
 
 
28
  os.makedirs("model", exist_ok=True)
29
  os.makedirs("animation", exist_ok=True)
30
 
31
+ # Extract animations from zip if available on startup
32
+ if os.path.exists(ANIMATION_ZIP_PATH):
33
+ print("Extracting animations...")
34
+ try:
35
+ with zipfile.ZipFile(ANIMATION_ZIP_PATH, 'r') as z:
36
+ z.extractall("animation")
37
+ except Exception as e:
38
+ print(f"Error extracting animations: {e}")
 
 
 
39
 
40
+ # Helper: Get list of animations from extracted directory
41
  def get_animation_list():
 
 
 
42
  try:
43
+ files = [f for f in os.listdir("animation") if f.endswith('.vrma')]
44
+ if files:
45
+ return files
46
  except Exception as e:
47
  print(f"Error reading animations: {e}")
48
+ return ["neutral2.vrma", "neutral3.vrma", "happy.vrma", "wave.vrma", "dance.vrma"]
49
 
50
  ANIMATIONS_LIST = get_animation_list()
51
  EMOTIONS = [
 
87
  "motion": "idle"
88
  }}"""
89
 
90
+ # Load local LLM
91
+ print(f"Loading local LLM: {MODEL_NAME}...")
92
+ device = "cuda" if torch.cuda.is_available() else "cpu"
93
+ tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
94
+ model = AutoModelForCausalLM.from_pretrained(
95
+ MODEL_NAME,
96
+ torch_dtype=torch.float16 if device == "cuda" else torch.float32,
97
+ device_map="auto" if device == "cuda" else None
98
+ )
99
+ # Use a fast pipeline
100
+ llm_pipeline = pipeline("text-generation", model=model, tokenizer=tokenizer)
101
+
102
  async def generate_speech(text, rate, pitch):
103
  communicate = edge_tts.Communicate(text, VOICE, rate=rate, pitch=pitch)
104
  filename = f"speech_{uuid.uuid4().hex}.mp3"
105
  await communicate.save(filename)
106
  return filename
107
 
108
+ # Synchronous GPU helper to perform LLM generation
109
  @spaces.GPU
110
  def generate_llm_response(prompt):
111
  outputs = llm_pipeline(prompt, max_new_tokens=256, do_sample=True, temperature=0.75)
112
  return outputs[0]["generated_text"][len(prompt):].strip()
113
 
114
+ # Async chat handler handles I/O and non-GPU tasks
115
  async def chat_func(user_input, history):
116
  if not user_input:
117
  return "", history, None, {}
 
125
 
126
  prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
127
 
128
+ # Run pipeline on the ZeroGPU helper
129
  response_text = generate_llm_response(prompt)
130
 
131
+ # Clean up JSON if LLM added markdown wrappers
132
  json_str = response_text
133
  if "```" in json_str:
134
  json_str = json_str.split("```")[1]
 
248
  <script type="importmap">
249
  {
250
  "imports": {
251
+ "three": "https://cdn.jsdelivr.net/npm/three@0.160.0/build/three.module.js",
252
+ "three/addons/": "https://cdn.jsdelivr.net/npm/three@0.160.0/examples/jsm/",
253
+ "@pixiv/three-vrm": "https://cdn.jsdelivr.net/npm/@pixiv/three-vrm@3.2.0/lib/three-vrm.module.min.js",
254
+ "@pixiv/three-vrm-animation": "https://cdn.jsdelivr.net/npm/@pixiv/three-vrm-animation@3.2.0/lib/three-vrm-animation.module.min.js"
255
  }
256
  }
257
  </script>
 
291
  let audioContext, analyser, dataArray;
292
  let isSpeaking = false;
293
  let blinkT = 0, nextBlink = 3, blinkV = 0;
294
+
295
+ // Store current expressions and state so the mixer doesn't reset them on update
296
+ let activeExpressions = { 'neutral': 1.0 };
297
+ let currentVrmaAction = null;
298
 
299
  const loader = new GLTFLoader();
300
  loader.register(p => new VRMLoaderPlugin(p));
301
  loader.register(p => new VRMAnimationLoaderPlugin(p));
302
 
303
+ // Dynamic VRMA Animation Player
304
+ function playVrma(vrmaName) {
305
+ if (!currentVrm || !mixer) return;
306
+ const vrmaUrl = `/gradio_api/file=animation/${vrmaName}`;
307
+ loader.load(vrmaUrl, (gltf) => {
308
+ const vrmAnimations = gltf.userData.vrmAnimations;
309
+ if (vrmAnimations && vrmAnimations.length > 0) {
310
+ if (currentVrmaAction) {
311
+ currentVrmaAction.stop();
312
+ mixer.uncacheClip(currentVrmaAction.getClip());
313
+ }
314
+ const clip = createVRMAnimationClip(vrmAnimations[0], currentVrm);
315
+ currentVrmaAction = mixer.clipAction(clip);
316
+ currentVrmaAction.play();
317
+ }
318
+ }, undefined, (err) => console.log("VRMA loading skipped or file missing:", vrmaName));
319
+ }
320
+
321
+ // Load VRM Model securely using /gradio_api/file=
322
+ loader.load('/gradio_api/file=model/Ani.vrm', (gltf) => {
323
  const vrm = gltf.userData.vrm;
324
  VRMUtils.rotateVRM0(vrm);
325
  scene.add(vrm.scene);
326
  currentVrm = vrm;
327
  mixer = new THREE.AnimationMixer(vrm.scene);
328
 
329
+ // Initial Pose Setup
330
  vrm.expressionManager.setValue('neutral', 1.0);
331
 
332
+ // Auto-reframe on face
333
  const head = vrm.humanoid.getRawBoneNode('head');
334
  if (head) {
335
  const worldPos = new THREE.Vector3();
 
337
  orbit.target.copy(worldPos);
338
  orbit.update();
339
  }
340
+
341
+ // Start default/idle animation
342
+ playVrma('neutral2.vrma');
343
  }, undefined, (e) => console.log("VRM load placeholder or missing. Use model/Ani.vrm"));
344
 
345
  function initAudioSync(audioElement) {
 
363
  if (currentVrm) {
364
  if (mixer) mixer.update(dt);
365
 
366
+ // Reapply facial expressions on every frame after mixer update to prevent resets
367
+ if (activeExpressions) {
368
+ for (const [key, val] of Object.entries(activeExpressions)) {
369
+ currentVrm.expressionManager.setValue(key, val);
370
+ }
371
+ }
372
+
373
+ // Natural dynamic Lip Sync
374
  if (isSpeaking && analyser) {
375
  analyser.getByteFrequencyData(dataArray);
376
  let sum = 0;
 
386
  currentVrm.expressionManager.setValue('ee', 0);
387
  }
388
 
389
+ // Natural Eye Blinking
390
+ const isEmotionActive = activeExpressions && (activeExpressions.happy > 0.5 || activeExpressions.surprised > 0.5 || activeExpressions.relaxed > 0.5);
391
  if (!isSpeaking && !isEmotionActive) {
392
  blinkT += dt;
393
  if (blinkT > nextBlink) {
 
428
  document.getElementById('speakDot').classList.remove('on');
429
  };
430
 
431
+ // Process dynamic animations returned from LLM
432
+ if (data.vrma) {
433
+ playVrma(data.vrma);
434
+ }
435
+
436
+ // Process active facial expression changes
437
  if (currentVrm && data.expressions) {
438
+ activeExpressions = {};
439
  ['happy','sad','angry','surprised','relaxed','neutral'].forEach(ex => {
440
  currentVrm.expressionManager.setValue(ex, 0);
441
  });
442
  for (const [key, val] of Object.entries(data.expressions)) {
443
+ activeExpressions[key] = val;
444
+ }
445
+ if (Object.keys(activeExpressions).length === 0) {
446
+ activeExpressions['neutral'] = 1.0;
447
  }
 
448
  }
449
  }
450
  });
 
457
  }
458
  """
459
 
460
+ # Construct a warning system in Gradio if the model file hasn't been uploaded yet
461
+ model_warning = ""
462
+ if not os.path.exists(VRM_MODEL_PATH):
463
+ model_warning = """
464
+ <div style="position: fixed; top: 15px; left: 50%; transform: translateX(-50%); z-index: 9999;
465
+ background: rgba(255, 75, 75, 0.95); color: white; padding: 15px 30px; border-radius: 12px;
466
+ box-shadow: 0 4px 20px rgba(0,0,0,0.4); font-family: sans-serif; text-align: center; max-width: 90%;
467
+ border: 2px dashed #ffcccc;">
468
+ <strong style="font-size: 1.1em;">⚠️ 3D Avatar File Missing!</strong><br>
469
+ Please upload your 3D avatar file to your Space and save it at: <code>model/Ani.vrm</code>.
470
+ </div>
471
+ """
472
+
473
  with gr.Blocks() as demo:
474
  history = gr.State([])
475
 
476
+ # Display the file placeholder overlay if necessary
477
+ if model_warning:
478
+ gr.HTML(model_warning)
479
+
480
  with gr.Group(elem_id="custom-viewport"):
481
  gr.HTML('<div id="c"></div><div id="speakDot"><div class="sdot"></div><div class="sdot"></div><div class="sdot"></div></div>')
482
 
 
499
  def on_submit(text, hist):
500
  return chat_func(text, hist)
501
 
502
+ # Clean Javascript parser supporting both old and new Gradio structures for passing file-paths
503
  update_js = """
504
  (text, hist, audio_path, data) => {
505
+ let audio_url = "";
506
  if (audio_path) {
507
+ if (typeof audio_path === 'string') {
508
+ audio_url = audio_path;
509
+ } else if (audio_path.url) {
510
+ audio_url = audio_path.url;
511
+ } else if (audio_path.path) {
512
+ audio_url = "/gradio_api/file=" + audio_path.path;
513
+ }
514
+ }
515
+ if (audio_url && data) {
516
  window.postMessage({
517
  type: 'vrm_update',
518
+ audio_url: audio_url,
519
  data: data
520
  }, '*');
521
  }
 
533
  demo.load(None, js=JS_CODE)
534
 
535
  if __name__ == "__main__":
536
+ # Declare "allowed_paths" with working directory to allow local file serving safely
537
+ demo.launch(
538
+ server_name="0.0.0.0",
539
+ server_port=7860,
540
+ css=CSS,
541
+ head=HTML_HEAD,
542
+ allowed_paths=["."]
543
+ )