Nafiul Huda commited on
Commit
7e26e37
·
verified ·
1 Parent(s): 85aa8ce

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +192 -172
app.py CHANGED
@@ -159,7 +159,7 @@ async def chat_func(user_input, history):
159
 
160
  # Custom UI CSS and JS
161
  CSS = """
162
- body { margin: 0; padding: 0; background-color: #ffe082 !important; overflow: hidden; font-family: sans-serif; }
163
  .gradio-container { max-width: 100% !important; border: none !important; padding: 0 !important; background: transparent !important; }
164
 
165
  #custom-viewport {
@@ -265,198 +265,217 @@ async () => {
265
  const { VRMAnimationLoaderPlugin, createVRMAnimationClip } = await import('@pixiv/three-vrm-animation');
266
  const { OrbitControls } = await import('three/addons/controls/OrbitControls.js');
267
 
268
- const scene = new THREE.Scene();
269
- scene.background = new THREE.Color(0xffe082);
270
- const camera = new THREE.PerspectiveCamera(45, window.innerWidth / window.innerHeight, 0.1, 100);
271
- camera.position.set(0, 1.4, 1.8);
272
-
273
- const renderer = new THREE.WebGLRenderer({ antialias: true, alpha: true, powerPreference: 'high-performance' });
274
- renderer.setSize(window.innerWidth, window.innerHeight);
275
- renderer.setPixelRatio(Math.min(window.devicePixelRatio, 2));
276
- renderer.outputColorSpace = THREE.SRGBColorSpace;
277
- document.getElementById('c').appendChild(renderer.domElement);
278
-
279
- const orbit = new OrbitControls(camera, renderer.domElement);
280
- orbit.target.set(0, 1.3, 0);
281
- orbit.enableDamping = true;
282
- orbit.update();
283
-
284
- scene.add(new THREE.AmbientLight(0xfff4e0, 0.5));
285
- const keyLight = new THREE.DirectionalLight(0xffffff, 1.2);
286
- keyLight.position.set(2, 4, 3);
287
- scene.add(keyLight);
288
-
289
- let currentVrm = null;
290
- let mixer = null;
291
- let audioContext, analyser, dataArray;
292
- let isSpeaking = false;
293
- let blinkT = 0, nextBlink = 3, blinkV = 0;
294
-
295
- // Store current expressions and state so the mixer doesn't reset them on update
296
- let activeExpressions = { 'neutral': 1.0 };
297
- let currentVrmaAction = null;
298
-
299
- const loader = new GLTFLoader();
300
- loader.register(p => new VRMLoaderPlugin(p));
301
- loader.register(p => new VRMAnimationLoaderPlugin(p));
302
-
303
- // Dynamic VRMA Animation Player
304
- function playVrma(vrmaName) {
305
- if (!currentVrm || !mixer) return;
306
- const vrmaUrl = `/gradio_api/file=animation/${vrmaName}`;
307
- loader.load(vrmaUrl, (gltf) => {
308
- const vrmAnimations = gltf.userData.vrmAnimations;
309
- if (vrmAnimations && vrmAnimations.length > 0) {
310
- if (currentVrmaAction) {
311
- currentVrmaAction.stop();
312
- mixer.uncacheClip(currentVrmaAction.getClip());
313
- }
314
- const clip = createVRMAnimationClip(vrmAnimations[0], currentVrm);
315
- currentVrmaAction = mixer.clipAction(clip);
316
- currentVrmaAction.play();
317
- }
318
- }, undefined, (err) => console.log("VRMA loading skipped or file missing:", vrmaName));
319
  }
320
 
321
- // Load VRM Model securely using /gradio_api/file=
322
- loader.load('/gradio_api/file=model/Ani.vrm', (gltf) => {
323
- const vrm = gltf.userData.vrm;
324
- VRMUtils.rotateVRM0(vrm);
325
- scene.add(vrm.scene);
326
- currentVrm = vrm;
327
- mixer = new THREE.AnimationMixer(vrm.scene);
328
-
329
- // Initial Pose Setup
330
- vrm.expressionManager.setValue('neutral', 1.0);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
331
 
332
- // Auto-reframe on face
333
- const head = vrm.humanoid.getRawBoneNode('head');
334
- if (head) {
335
- const worldPos = new THREE.Vector3();
336
- head.getWorldPosition(worldPos);
337
- orbit.target.copy(worldPos);
338
- orbit.update();
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
339
  }
340
 
341
- // Start default/idle animation
342
- playVrma('neutral2.vrma');
343
- }, undefined, (e) => console.log("VRM load placeholder or missing. Use model/Ani.vrm"));
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
344
 
345
- function initAudioSync(audioElement) {
346
- if (!audioContext) {
347
- audioContext = new (window.AudioContext || window.webkitAudioContext)();
348
- analyser = audioContext.createAnalyser();
349
- analyser.fftSize = 256;
350
- dataArray = new Uint8Array(analyser.frequencyBinCount);
 
 
 
 
 
351
  }
352
- if (audioContext.state === 'suspended') audioContext.resume();
353
- const source = audioContext.createMediaElementSource(audioElement);
354
- source.connect(analyser);
355
- analyser.connect(audioContext.destination);
356
- }
357
 
358
- const clock = new THREE.Clock();
359
- function animate() {
360
- requestAnimationFrame(animate);
361
- const dt = Math.min(clock.getDelta(), 0.1);
362
-
363
- if (currentVrm) {
364
- if (mixer) mixer.update(dt);
365
 
366
- // Reapply facial expressions on every frame after mixer update to prevent resets
367
- if (activeExpressions) {
368
- for (const [key, val] of Object.entries(activeExpressions)) {
369
- currentVrm.expressionManager.setValue(key, val);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
370
  }
371
- }
372
-
373
- // Natural dynamic Lip Sync
374
- if (isSpeaking && analyser) {
375
- analyser.getByteFrequencyData(dataArray);
376
- let sum = 0;
377
- for (let i = 0; i < 32; i++) sum += dataArray[i];
378
- const level = sum / 32 / 255;
379
- const open = Math.min(level * 4.5, 1.2);
380
- currentVrm.expressionManager.setValue('aa', open);
381
- currentVrm.expressionManager.setValue('oh', open * 0.4);
382
- currentVrm.expressionManager.setValue('ee', open * 0.2);
383
- } else {
384
- currentVrm.expressionManager.setValue('aa', 0);
385
- currentVrm.expressionManager.setValue('oh', 0);
386
- currentVrm.expressionManager.setValue('ee', 0);
387
- }
388
 
389
- // Natural Eye Blinking
390
- const isEmotionActive = activeExpressions && (activeExpressions.happy > 0.5 || activeExpressions.surprised > 0.5 || activeExpressions.relaxed > 0.5);
391
- if (!isSpeaking && !isEmotionActive) {
392
- blinkT += dt;
393
- if (blinkT > nextBlink) {
394
- if (blinkV === 0) blinkV = 1;
395
- if (blinkV === 1) {
396
- let v = currentVrm.expressionManager.getValue('blink') || 0;
397
- v += dt * 12;
398
- if (v >= 1) { v = 1; blinkV = -1; }
399
- currentVrm.expressionManager.setValue('blink', v);
400
- } else {
401
- let v = currentVrm.expressionManager.getValue('blink') || 1;
402
- v -= dt * 12;
403
- if (v <= 0) { v = 0; blinkV = 0; blinkT = 0; nextBlink = 2 + Math.random() * 5; }
404
- currentVrm.expressionManager.setValue('blink', v);
 
405
  }
 
 
406
  }
407
- } else {
408
- currentVrm.expressionManager.setValue('blink', 0);
409
- }
410
 
411
- currentVrm.update(dt);
412
- }
413
- orbit.update();
414
- renderer.render(scene, camera);
415
- }
416
- animate();
417
-
418
- window.addEventListener('message', (e) => {
419
- if (e.data.type === 'vrm_update') {
420
- const { audio_url, data } = e.data;
421
- const audio = new Audio(audio_url);
422
- initAudioSync(audio);
423
- isSpeaking = true;
424
- document.getElementById('speakDot').classList.add('on');
425
- audio.play();
426
- audio.onended = () => {
427
- isSpeaking = false;
428
- document.getElementById('speakDot').classList.remove('on');
429
- };
430
-
431
- // Process dynamic animations returned from LLM
432
- if (data.vrma) {
433
- playVrma(data.vrma);
434
  }
435
-
436
- // Process active facial expression changes
437
- if (currentVrm && data.expressions) {
438
- activeExpressions = {};
439
- ['happy','sad','angry','surprised','relaxed','neutral'].forEach(ex => {
440
- currentVrm.expressionManager.setValue(ex, 0);
441
- });
442
- for (const [key, val] of Object.entries(data.expressions)) {
443
- activeExpressions[key] = val;
 
 
 
 
 
 
 
 
 
 
 
 
444
  }
445
- if (Object.keys(activeExpressions).length === 0) {
446
- activeExpressions['neutral'] = 1.0;
 
 
 
 
 
 
 
 
 
 
 
447
  }
448
  }
449
- }
450
- });
451
 
452
- window.addEventListener('resize', () => {
453
- camera.aspect = window.innerWidth / window.innerHeight;
454
- camera.updateProjectionMatrix();
455
- renderer.setSize(window.innerWidth, window.innerHeight);
456
- });
 
 
 
457
  }
458
  """
459
 
 
 
 
 
 
460
  # Construct a warning system in Gradio if the model file hasn't been uploaded yet
461
  model_warning = ""
462
  if not os.path.exists(VRM_MODEL_PATH):
@@ -530,7 +549,8 @@ with gr.Blocks() as demo:
530
  None, [text_input, history, audio_out, data_out], js=update_js
531
  )
532
 
533
- demo.load(None, js=JS_CODE)
 
534
 
535
  if __name__ == "__main__":
536
  # Declare "allowed_paths" with working directory to allow local file serving safely
 
159
 
160
  # Custom UI CSS and JS
161
  CSS = """
162
+ body { margin: 0; padding: 0; background-color: #ffe082 !important; overflow: hidden !important; font-family: sans-serif; }
163
  .gradio-container { max-width: 100% !important; border: none !important; padding: 0 !important; background: transparent !important; }
164
 
165
  #custom-viewport {
 
265
  const { VRMAnimationLoaderPlugin, createVRMAnimationClip } = await import('@pixiv/three-vrm-animation');
266
  const { OrbitControls } = await import('three/addons/controls/OrbitControls.js');
267
 
268
+ // Poll until DOM container element is successfully rendered by Gradio
269
+ function init() {
270
+ const container = document.getElementById('c');
271
+ if (!container) {
272
+ setTimeout(init, 100);
273
+ return;
274
+ }
275
+ setupApp(container);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
276
  }
277
 
278
+ function setupApp(container) {
279
+ const scene = new THREE.Scene();
280
+ scene.background = new THREE.Color(0xffe082);
281
+ const camera = new THREE.PerspectiveCamera(45, window.innerWidth / window.innerHeight, 0.1, 100);
282
+ camera.position.set(0, 1.4, 1.8);
283
+
284
+ const renderer = new THREE.WebGLRenderer({ antialias: true, alpha: true, powerPreference: 'high-performance' });
285
+ renderer.setSize(window.innerWidth, window.innerHeight);
286
+ renderer.setPixelRatio(Math.min(window.devicePixelRatio, 2));
287
+ renderer.outputColorSpace = THREE.SRGBColorSpace;
288
+ container.appendChild(renderer.domElement);
289
+
290
+ const orbit = new OrbitControls(camera, renderer.domElement);
291
+ orbit.target.set(0, 1.3, 0);
292
+ orbit.enableDamping = true;
293
+ orbit.update();
294
+
295
+ scene.add(new THREE.AmbientLight(0xfff4e0, 0.5));
296
+ const keyLight = new THREE.DirectionalLight(0xffffff, 1.2);
297
+ keyLight.position.set(2, 4, 3);
298
+ scene.add(keyLight);
299
+
300
+ let currentVrm = null;
301
+ let mixer = null;
302
+ let audioContext, analyser, dataArray;
303
+ let isSpeaking = false;
304
+ let blinkT = 0, nextBlink = 3, blinkV = 0;
305
 
306
+ // Store current expressions and state so the mixer doesn't reset them on update
307
+ let activeExpressions = { 'neutral': 1.0 };
308
+ let currentVrmaAction = null;
309
+
310
+ const loader = new GLTFLoader();
311
+ loader.register(p => new VRMLoaderPlugin(p));
312
+ loader.register(p => new VRMAnimationLoaderPlugin(p));
313
+
314
+ // Dynamic VRMA Animation Player
315
+ function playVrma(vrmaName) {
316
+ if (!currentVrm || !mixer) return;
317
+ const vrmaUrl = `/gradio_api/file=__ANIMATION_ABS_DIR__/${vrmaName}`;
318
+ loader.load(vrmaUrl, (gltf) => {
319
+ const vrmAnimations = gltf.userData.vrmAnimations;
320
+ if (vrmAnimations && vrmAnimations.length > 0) {
321
+ if (currentVrmaAction) {
322
+ currentVrmaAction.stop();
323
+ mixer.uncacheClip(currentVrmaAction.getClip());
324
+ }
325
+ const clip = createVRMAnimationClip(vrmAnimations[0], currentVrm);
326
+ currentVrmaAction = mixer.clipAction(clip);
327
+ currentVrmaAction.play();
328
+ }
329
+ }, undefined, (err) => console.log("VRMA loading skipped or file missing:", vrmaName));
330
  }
331
 
332
+ // Load VRM Model securely using resolved absolute path
333
+ loader.load('/gradio_api/file=__VRM_ABS_PATH__', (gltf) => {
334
+ const vrm = gltf.userData.vrm;
335
+ VRMUtils.rotateVRM0(vrm);
336
+ scene.add(vrm.scene);
337
+ currentVrm = vrm;
338
+ mixer = new THREE.AnimationMixer(vrm.scene);
339
+
340
+ // Initial Pose Setup
341
+ vrm.expressionManager.setValue('neutral', 1.0);
342
+
343
+ // Auto-reframe on face
344
+ const head = vrm.humanoid.getRawBoneNode('head');
345
+ if (head) {
346
+ const worldPos = new THREE.Vector3();
347
+ head.getWorldPosition(worldPos);
348
+ orbit.target.copy(worldPos);
349
+ orbit.update();
350
+ }
351
+
352
+ // Start default/idle animation
353
+ playVrma('neutral2.vrma');
354
+ }, undefined, (e) => console.log("VRM load placeholder or missing. Use model/Ani.vrm"));
355
 
356
+ function initAudioSync(audioElement) {
357
+ if (!audioContext) {
358
+ audioContext = new (window.AudioContext || window.webkitAudioContext)();
359
+ analyser = audioContext.createAnalyser();
360
+ analyser.fftSize = 256;
361
+ dataArray = new Uint8Array(analyser.frequencyBinCount);
362
+ }
363
+ if (audioContext.state === 'suspended') audioContext.resume();
364
+ const source = audioContext.createMediaElementSource(audioElement);
365
+ source.connect(analyser);
366
+ analyser.connect(audioContext.destination);
367
  }
 
 
 
 
 
368
 
369
+ const clock = new THREE.Clock();
370
+ function animate() {
371
+ requestAnimationFrame(animate);
372
+ const dt = Math.min(clock.getDelta(), 0.1);
 
 
 
373
 
374
+ if (currentVrm) {
375
+ if (mixer) mixer.update(dt);
376
+
377
+ // Reapply facial expressions on every frame after mixer update to prevent resets
378
+ if (activeExpressions) {
379
+ for (const [key, val] of Object.entries(activeExpressions)) {
380
+ currentVrm.expressionManager.setValue(key, val);
381
+ }
382
+ }
383
+
384
+ // Natural dynamic Lip Sync
385
+ if (isSpeaking && analyser) {
386
+ analyser.getByteFrequencyData(dataArray);
387
+ let sum = 0;
388
+ for (let i = 0; i < 32; i++) sum += dataArray[i];
389
+ const level = sum / 32 / 255;
390
+ const open = Math.min(level * 4.5, 1.2);
391
+ currentVrm.expressionManager.setValue('aa', open);
392
+ currentVrm.expressionManager.setValue('oh', open * 0.4);
393
+ currentVrm.expressionManager.setValue('ee', open * 0.2);
394
+ } else {
395
+ currentVrm.expressionManager.setValue('aa', 0);
396
+ currentVrm.expressionManager.setValue('oh', 0);
397
+ currentVrm.expressionManager.setValue('ee', 0);
398
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
399
 
400
+ // Natural Eye Blinking
401
+ const isEmotionActive = activeExpressions && (activeExpressions.happy > 0.5 || activeExpressions.surprised > 0.5 || activeExpressions.relaxed > 0.5);
402
+ if (!isSpeaking && !isEmotionActive) {
403
+ blinkT += dt;
404
+ if (blinkT > nextBlink) {
405
+ if (blinkV === 0) blinkV = 1;
406
+ if (blinkV === 1) {
407
+ let v = currentVrm.expressionManager.getValue('blink') || 0;
408
+ v += dt * 12;
409
+ if (v >= 1) { v = 1; blinkV = -1; }
410
+ currentVrm.expressionManager.setValue('blink', v);
411
+ } else {
412
+ let v = currentVrm.expressionManager.getValue('blink') || 1;
413
+ v -= dt * 12;
414
+ if (v <= 0) { v = 0; blinkV = 0; blinkT = 0; nextBlink = 2 + Math.random() * 5; }
415
+ currentVrm.expressionManager.setValue('blink', v);
416
+ }
417
  }
418
+ } else {
419
+ currentVrm.expressionManager.setValue('blink', 0);
420
  }
 
 
 
421
 
422
+ currentVrm.update(dt);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
423
  }
424
+ orbit.update();
425
+ renderer.render(scene, camera);
426
+ }
427
+ animate();
428
+
429
+ window.addEventListener('message', (e) => {
430
+ if (e.data.type === 'vrm_update') {
431
+ const { audio_url, data } = e.data;
432
+ const audio = new Audio(audio_url);
433
+ initAudioSync(audio);
434
+ isSpeaking = true;
435
+ document.getElementById('speakDot').classList.add('on');
436
+ audio.play();
437
+ audio.onended = () => {
438
+ isSpeaking = false;
439
+ document.getElementById('speakDot').classList.remove('on');
440
+ };
441
+
442
+ // Process dynamic animations returned from LLM
443
+ if (data.vrma) {
444
+ playVrma(data.vrma);
445
  }
446
+
447
+ // Process active facial expression changes
448
+ if (currentVrm && data.expressions) {
449
+ activeExpressions = {};
450
+ ['happy','sad','angry','surprised','relaxed','neutral'].forEach(ex => {
451
+ currentVrm.expressionManager.setValue(ex, 0);
452
+ });
453
+ for (const [key, val] of Object.entries(data.expressions)) {
454
+ activeExpressions[key] = val;
455
+ }
456
+ if (Object.keys(activeExpressions).length === 0) {
457
+ activeExpressions['neutral'] = 1.0;
458
+ }
459
  }
460
  }
461
+ });
 
462
 
463
+ window.addEventListener('resize', () => {
464
+ camera.aspect = window.innerWidth / window.innerHeight;
465
+ camera.updateProjectionMatrix();
466
+ renderer.setSize(window.innerWidth, window.innerHeight);
467
+ });
468
+ }
469
+
470
+ init();
471
  }
472
  """
473
 
474
+ # Dynamically replace path placeholders with fully resolved absolute system paths
475
+ vrm_abs_path = os.path.abspath(VRM_MODEL_PATH)
476
+ animation_abs_dir = os.path.abspath("animation")
477
+ js_code_final = JS_CODE.replace("__VRM_ABS_PATH__", vrm_abs_path).replace("__ANIMATION_ABS_DIR__", animation_abs_dir)
478
+
479
  # Construct a warning system in Gradio if the model file hasn't been uploaded yet
480
  model_warning = ""
481
  if not os.path.exists(VRM_MODEL_PATH):
 
549
  None, [text_input, history, audio_out, data_out], js=update_js
550
  )
551
 
552
+ # Run polling loop inside load handler
553
+ demo.load(fn=None, inputs=None, outputs=None, js=js_code_final)
554
 
555
  if __name__ == "__main__":
556
  # Declare "allowed_paths" with working directory to allow local file serving safely