Nanny7 commited on
Commit
32c4979
·
1 Parent(s): bb6135e

Update echo-tutor features and UI

Browse files
static/js/features.js CHANGED
@@ -486,17 +486,19 @@ window.EchoFeatures = (function () {
486
  // Simple VAD (Voice Activity Detection) for auto-stop
487
  if (onSilence) {
488
  try {
489
- const audioCtx = new (window.AudioContext || window.webkitAudioContext)();
 
 
490
  const source = audioCtx.createMediaStreamSource(stream);
491
  const analyser = audioCtx.createAnalyser();
492
  analyser.fftSize = 512;
493
  analyser.smoothingTimeConstant = 0.1;
494
  source.connect(analyser);
495
-
496
  const dataArray = new Uint8Array(analyser.frequencyBinCount);
497
  let silenceStart = Date.now();
498
  let hasSpoken = false;
499
-
500
  const checkSilence = () => {
501
  if (!this.recording) {
502
  audioCtx.close().catch(() => {});
@@ -508,16 +510,19 @@ window.EchoFeatures = (function () {
508
  sum += dataArray[i];
509
  }
510
  const average = sum / dataArray.length;
511
-
512
- if (average > 15) { // Threshold for speech
 
513
  hasSpoken = true;
514
  silenceStart = Date.now();
515
  } else {
516
- if (hasSpoken && Date.now() - silenceStart > 2000) { // 2s silence after speech
 
517
  onSilence();
518
  audioCtx.close().catch(() => {});
519
  return;
520
- } else if (!hasSpoken && Date.now() - silenceStart > 10000) { // 10s silence without speech
 
521
  onSilence();
522
  audioCtx.close().catch(() => {});
523
  return;
@@ -543,7 +548,7 @@ window.EchoFeatures = (function () {
543
  resolve(null);
544
  return;
545
  }
546
-
547
  const finish = () => {
548
  this.recording = false;
549
  if (this.recorder && this.recorder.stream) {
@@ -560,7 +565,7 @@ window.EchoFeatures = (function () {
560
  };
561
 
562
  this.recorder.onstop = finish;
563
-
564
  try {
565
  if (this.recorder.state !== 'inactive') {
566
  this.recorder.stop();
 
486
  // Simple VAD (Voice Activity Detection) for auto-stop
487
  if (onSilence) {
488
  try {
489
+ const audioCtx = new (
490
+ window.AudioContext || window.webkitAudioContext
491
+ )();
492
  const source = audioCtx.createMediaStreamSource(stream);
493
  const analyser = audioCtx.createAnalyser();
494
  analyser.fftSize = 512;
495
  analyser.smoothingTimeConstant = 0.1;
496
  source.connect(analyser);
497
+
498
  const dataArray = new Uint8Array(analyser.frequencyBinCount);
499
  let silenceStart = Date.now();
500
  let hasSpoken = false;
501
+
502
  const checkSilence = () => {
503
  if (!this.recording) {
504
  audioCtx.close().catch(() => {});
 
510
  sum += dataArray[i];
511
  }
512
  const average = sum / dataArray.length;
513
+
514
+ if (average > 15) {
515
+ // Threshold for speech
516
  hasSpoken = true;
517
  silenceStart = Date.now();
518
  } else {
519
+ if (hasSpoken && Date.now() - silenceStart > 2000) {
520
+ // 2s silence after speech
521
  onSilence();
522
  audioCtx.close().catch(() => {});
523
  return;
524
+ } else if (!hasSpoken && Date.now() - silenceStart > 10000) {
525
+ // 10s silence without speech
526
  onSilence();
527
  audioCtx.close().catch(() => {});
528
  return;
 
548
  resolve(null);
549
  return;
550
  }
551
+
552
  const finish = () => {
553
  this.recording = false;
554
  if (this.recorder && this.recorder.stream) {
 
565
  };
566
 
567
  this.recorder.onstop = finish;
568
+
569
  try {
570
  if (this.recorder.state !== 'inactive') {
571
  this.recorder.stop();
static/js/script.js CHANGED
@@ -1384,31 +1384,42 @@ document.addEventListener('DOMContentLoaded', () => {
1384
  const mouthBars = [1, 2, 3, 4, 5]
1385
  .map((i) => $('mouth-bar-' + i))
1386
  .filter(Boolean);
 
1387
  function startLipSync() {
1388
- stopLipSync();
1389
- lipSyncInterval = setInterval(() => {
1390
- mouthBars.forEach((bar, i) => {
1391
- bar.style.height =
1392
- (i === 2 ? 8 : 4) + Math.random() * (i === 2 ? 6 : 4) + 'px';
 
 
 
 
 
 
 
 
 
 
 
 
 
1393
  });
1394
- }, 80);
1395
- }
1396
- function stopLipSync() {
1397
- if (lipSyncInterval) {
1398
- clearInterval(lipSyncInterval);
1399
- lipSyncInterval = null;
1400
  }
1401
- [3, 4, 5, 4, 3].forEach((h, i) => {
1402
- if (mouthBars[i]) mouthBars[i].style.height = h + 'px';
1403
- });
1404
- }
1405
-
1406
- // Emotion system
1407
- function showEmotion(emoji, dur = 2000) {
1408
- if (!emotionIndicator) return;
1409
  emotionIndicator.textContent = emoji;
1410
  emotionIndicator.classList.add('visible');
1411
- setTimeout(() => emotionIndicator.classList.remove('visible'), dur);
 
 
 
 
 
 
 
 
 
 
 
1412
  }
1413
 
1414
  function reactToEmotion(text) {
@@ -1463,13 +1474,15 @@ document.addEventListener('DOMContentLoaded', () => {
1463
  }, 5000);
1464
 
1465
  try {
1466
- const stream =
1467
- await window.EchoFeatures.Whisper.start(getSttLanguageCode(), () => {
1468
- if (whisperRecording) {
1469
- console.log('Silence detected, auto-stopping Whisper');
1470
- stopWhisperRecordingAndSend();
1471
- }
1472
- });
 
 
1473
  if (!stream) {
1474
  showToast('Microphone unavailable', 'error');
1475
  setAvatarState('idle');
 
1384
  const mouthBars = [1, 2, 3, 4, 5]
1385
  .map((i) => $('mouth-bar-' + i))
1386
  .filter(Boolean);
1387
+
1388
  function startLipSync() {
1389
+ stopLipSync();
1390
+ if (window.setVRMLipSync) window.setVRMLipSync(true);
1391
+ lipSyncInterval = setInterval(() => {
1392
+ mouthBars.forEach((bar, i) => {
1393
+ if(bar) bar.style.height =
1394
+ (i === 2 ? 8 : 4) + Math.random() * (i === 2 ? 6 : 4) + 'px';
1395
+ });
1396
+ }, 80);
1397
+ }
1398
+
1399
+ function stopLipSync() {
1400
+ if (window.setVRMLipSync) window.setVRMLipSync(false);
1401
+ if (lipSyncInterval) {
1402
+ clearInterval(lipSyncInterval);
1403
+ lipSyncInterval = null;
1404
+ }
1405
+ [3, 4, 5, 4, 3].forEach((h, i) => {
1406
+ if (mouthBars[i]) mouthBars[i].style.height = h + 'px';
1407
  });
 
 
 
 
 
 
1408
  }
 
 
 
 
 
 
 
 
1409
  emotionIndicator.textContent = emoji;
1410
  emotionIndicator.classList.add('visible');
1411
+
1412
+ if (window.setVRMEmotion) {
1413
+ if (emoji === '🌟' || emoji === '😊' || emoji === '🎉') window.setVRMEmotion('happy');
1414
+ else if (emoji === '🤔' || emoji === '🧐') window.setVRMEmotion('neutral');
1415
+ else if (emoji === '💡') window.setVRMEmotion('relaxed');
1416
+ else window.setVRMEmotion('neutral');
1417
+ }
1418
+
1419
+ setTimeout(() => {
1420
+ emotionIndicator.classList.remove('visible');
1421
+ if (window.setVRMEmotion) window.setVRMEmotion('neutral');
1422
+ }, dur);
1423
  }
1424
 
1425
  function reactToEmotion(text) {
 
1474
  }, 5000);
1475
 
1476
  try {
1477
+ const stream = await window.EchoFeatures.Whisper.start(
1478
+ getSttLanguageCode(),
1479
+ () => {
1480
+ if (whisperRecording) {
1481
+ console.log('Silence detected, auto-stopping Whisper');
1482
+ stopWhisperRecordingAndSend();
1483
+ }
1484
+ },
1485
+ );
1486
  if (!stream) {
1487
  showToast('Microphone unavailable', 'error');
1488
  setAvatarState('idle');
static/js/vrm-avatar.js ADDED
@@ -0,0 +1,148 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import * as THREE from 'three';
2
+ import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js';
3
+ import { VRMLoaderPlugin, VRMUtils } from '@pixiv/three-vrm';
4
+
5
+ // Global variables
6
+ let currentVrm = null;
7
+ let isSpeaking = false;
8
+ let currentEmotion = 'neutral';
9
+
10
+ function initVRM() {
11
+ const container = document.getElementById('avatar-container');
12
+ const orb = document.getElementById('avatar-orb');
13
+
14
+ if (!container) return;
15
+
16
+ // Hide the old CSS orb
17
+ if (orb) {
18
+ orb.style.display = 'none';
19
+ }
20
+
21
+ const scene = new THREE.Scene();
22
+
23
+ // Camera
24
+ const camera = new THREE.PerspectiveCamera(30.0, container.clientWidth / container.clientHeight, 0.1, 20.0);
25
+ camera.position.set(0.0, 1.4, 1.5); // Focus on the face
26
+
27
+ // Renderer
28
+ const renderer = new THREE.WebGLRenderer({ alpha: true, antialias: true });
29
+ renderer.setSize(container.clientWidth, container.clientHeight);
30
+ renderer.setPixelRatio(window.devicePixelRatio);
31
+ // Make the canvas fit the container
32
+ renderer.domElement.style.position = 'absolute';
33
+ renderer.domElement.style.top = '0';
34
+ renderer.domElement.style.left = '0';
35
+ renderer.domElement.style.width = '100%';
36
+ renderer.domElement.style.height = '100%';
37
+ renderer.domElement.style.zIndex = '10';
38
+ container.appendChild(renderer.domElement);
39
+
40
+ // Light
41
+ const light = new THREE.DirectionalLight(0xffffff, Math.PI);
42
+ light.position.set(1.0, 1.0, 1.0).normalize();
43
+ scene.add(light);
44
+
45
+ // Load VRM
46
+ const loader = new GLTFLoader();
47
+ loader.crossOrigin = 'anonymous';
48
+
49
+ loader.register((parser) => {
50
+ return new VRMLoaderPlugin(parser);
51
+ });
52
+
53
+ // Using a sample VRM model from three-vrm examples
54
+ const vrmUrl = 'https://cdn.jsdelivr.net/gh/pixiv/three-vrm@main/packages/three-vrm/examples/models/VRM1_Constraint_Twist_Sample.vrm';
55
+
56
+ loader.load(
57
+ vrmUrl,
58
+ (gltf) => {
59
+ const vrm = gltf.userData.vrm;
60
+
61
+ // Disable frustum culling
62
+ VRMUtils.removeUnnecessaryJoints(gltf.scene);
63
+ VRMUtils.removeUnnecessaryVertices(gltf.scene);
64
+
65
+ scene.add(vrm.scene);
66
+ currentVrm = vrm;
67
+
68
+ // Rotate model to face camera
69
+ vrm.scene.rotation.y = Math.PI;
70
+
71
+ console.log('VRM loaded successfully');
72
+ },
73
+ (progress) => console.log('Loading model...', 100.0 * (progress.loaded / progress.total), '%'),
74
+ (error) => console.error('Error loading VRM:', error)
75
+ );
76
+
77
+ // Animation Loop
78
+ const clock = new THREE.Clock();
79
+
80
+ function animate() {
81
+ requestAnimationFrame(animate);
82
+
83
+ const deltaTime = clock.getDelta();
84
+
85
+ if (currentVrm) {
86
+ // Update VRM components
87
+ currentVrm.update(deltaTime);
88
+
89
+ // Handle Lip Sync (Fake 3D Lip Sync)
90
+ if (isSpeaking) {
91
+ // Randomly open and close mouth to simulate talking
92
+ const mouthOpen = Math.random() * 0.8 + 0.2; // Random value between 0.2 and 1.0
93
+ currentVrm.expressionManager.setValue('aa', mouthOpen);
94
+ } else {
95
+ // Close mouth when not speaking
96
+ currentVrm.expressionManager.setValue('aa', 0);
97
+ }
98
+
99
+ // Handle Emotions (Blinking, etc.)
100
+ // Add a simple blink every few seconds
101
+ const time = clock.getElapsedTime();
102
+ if (Math.sin(time * 2) > 0.95) {
103
+ currentVrm.expressionManager.setValue('blink', 1);
104
+ } else {
105
+ currentVrm.expressionManager.setValue('blink', 0);
106
+ }
107
+ }
108
+
109
+ renderer.render(scene, camera);
110
+ }
111
+
112
+ animate();
113
+
114
+ // Handle window resize
115
+ window.addEventListener('resize', () => {
116
+ if (!container) return;
117
+ camera.aspect = container.clientWidth / container.clientHeight;
118
+ camera.updateProjectionMatrix();
119
+ renderer.setSize(container.clientWidth, container.clientHeight);
120
+ });
121
+ }
122
+
123
+ // Initialize when DOM is ready
124
+ if (document.readyState === 'loading') {
125
+ document.addEventListener('DOMContentLoaded', initVRM);
126
+ } else {
127
+ initVRM();
128
+ }
129
+
130
+ // Expose functions to global scope so script.js can call them
131
+ window.setVRMLipSync = function(speaking) {
132
+ isSpeaking = speaking;
133
+ };
134
+
135
+ window.setVRMEmotion = function(emotion) {
136
+ currentEmotion = emotion;
137
+ if (currentVrm) {
138
+ // Reset all expressions
139
+ ['neutral', 'happy', 'angry', 'sad', 'relaxed'].forEach(exp => {
140
+ currentVrm.expressionManager.setValue(exp, 0);
141
+ });
142
+
143
+ // Set new expression if valid
144
+ if (['neutral', 'happy', 'angry', 'sad', 'relaxed'].includes(emotion)) {
145
+ currentVrm.expressionManager.setValue(emotion, 1);
146
+ }
147
+ }
148
+ };
templates/index.html CHANGED
@@ -12,6 +12,18 @@
12
  content="Practice languages with Echo, your AI-powered tutor. Real-time conversation, corrections, exercises, and progress tracking."
13
  />
14
  <meta name="theme-color" content="#0a0e1a" />
 
 
 
 
 
 
 
 
 
 
 
 
15
  <meta name="apple-mobile-web-app-capable" content="yes" />
16
  <meta
17
  name="apple-mobile-web-app-status-bar-style"
@@ -916,6 +928,7 @@
916
  <script src="{{ url_for('static', filename='js/features.js') }}"></script>
917
  <script src="{{ url_for('static', filename='js/stories-flashcards.js') }}"></script>
918
  <script src="{{ url_for('static', filename='js/script.js') }}"></script>
 
919
  <script>
920
  if ('serviceWorker' in navigator) {
921
  window.addEventListener('load', () => {
 
12
  content="Practice languages with Echo, your AI-powered tutor. Real-time conversation, corrections, exercises, and progress tracking."
13
  />
14
  <meta name="theme-color" content="#0a0e1a" />
15
+
16
+ <!-- Three.js and VRM Import Map -->
17
+ <script type="importmap">
18
+ {
19
+ "imports": {
20
+ "three": "https://unpkg.com/three@0.160.0/build/three.module.js",
21
+ "three/addons/": "https://unpkg.com/three@0.160.0/examples/jsm/",
22
+ "@pixiv/three-vrm": "https://unpkg.com/@pixiv/three-vrm@2.0.0/lib/three-vrm.module.js"
23
+ }
24
+ }
25
+ </script>
26
+
27
  <meta name="apple-mobile-web-app-capable" content="yes" />
28
  <meta
29
  name="apple-mobile-web-app-status-bar-style"
 
928
  <script src="{{ url_for('static', filename='js/features.js') }}"></script>
929
  <script src="{{ url_for('static', filename='js/stories-flashcards.js') }}"></script>
930
  <script src="{{ url_for('static', filename='js/script.js') }}"></script>
931
+ <script type="module" src="{{ url_for('static', filename='js/vrm-avatar.js') }}"></script>
932
  <script>
933
  if ('serviceWorker' in navigator) {
934
  window.addEventListener('load', () => {