Nafiul Huda commited on
Commit
018703e
·
verified ·
1 Parent(s): c6b055b

Upload viewer.html

Browse files
Files changed (1) hide show
  1. viewer.html +523 -0
viewer.html ADDED
@@ -0,0 +1,523 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <!DOCTYPE html>
2
+ <html>
3
+ <head>
4
+ <meta charset="utf-8" />
5
+ <title>3D VRM Viewer Iframe</title>
6
+ <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no, viewport-fit=cover">
7
+ <style>
8
+ body {
9
+ margin: 0;
10
+ padding: 0;
11
+ background-color: #ffe082;
12
+ overflow: hidden;
13
+ font-family: sans-serif;
14
+ }
15
+ #c {
16
+ width: 100vw;
17
+ height: 100vh;
18
+ display: block;
19
+ }
20
+ #speakDot {
21
+ position: fixed;
22
+ bottom: 100px;
23
+ left: 50%;
24
+ transform: translateX(-50%);
25
+ display: flex;
26
+ gap: 6px;
27
+ z-index: 100;
28
+ opacity: 0;
29
+ transition: opacity 0.3s;
30
+ }
31
+ #speakDot.on {
32
+ opacity: 1;
33
+ }
34
+ .sdot {
35
+ width: 6px;
36
+ height: 6px;
37
+ border-radius: 50%;
38
+ background: #6cf;
39
+ animation: sdotBounce 0.6s infinite alternate;
40
+ }
41
+ .sdot:nth-child(2) { animation-delay: 0.15s; }
42
+ .sdot:nth-child(3) { animation-delay: 0.3s; }
43
+ @keyframes sdotBounce {
44
+ from { transform: scaleY(1); }
45
+ to { transform: scaleY(2.2); }
46
+ }
47
+
48
+ #load {
49
+ position: fixed;
50
+ top: 15px;
51
+ left: 50%;
52
+ transform: translateX(-50%);
53
+ background: rgba(15, 18, 32, 0.92);
54
+ padding: 8px 20px;
55
+ border-radius: 14px;
56
+ font-size: 12px;
57
+ color: #6cf;
58
+ z-index: 101;
59
+ backdrop-filter: blur(8px);
60
+ opacity: 0;
61
+ transition: opacity 0.2s;
62
+ pointer-events: none;
63
+ white-space: nowrap;
64
+ border: 1px solid rgba(108, 204, 255, 0.2);
65
+ }
66
+ #load.on {
67
+ opacity: 1;
68
+ }
69
+
70
+ #micBtn {
71
+ position: fixed;
72
+ left: 15px;
73
+ bottom: 15px;
74
+ width: 48px;
75
+ height: 48px;
76
+ border-radius: 50%;
77
+ border: none;
78
+ font-size: 18px;
79
+ cursor: pointer;
80
+ display: flex;
81
+ align-items: center;
82
+ justify-content: center;
83
+ background: rgba(15, 18, 32, 0.92);
84
+ color: #6cf;
85
+ border: 1px solid rgba(108, 204, 255, 0.3);
86
+ backdrop-filter: blur(12px);
87
+ z-index: 102;
88
+ transition: all 0.12s;
89
+ }
90
+ </style>
91
+
92
+ <script type="importmap">
93
+ {
94
+ "imports": {
95
+ "three": "https://cdn.jsdelivr.net/npm/three@0.160.0/build/three.module.js",
96
+ "three/addons/": "https://cdn.jsdelivr.net/npm/three@0.160.0/examples/jsm/",
97
+ "@pixiv/three-vrm": "https://cdn.jsdelivr.net/npm/@pixiv/three-vrm@3.2.0/lib/three-vrm.module.min.js",
98
+ "@pixiv/three-vrm-animation": "https://cdn.jsdelivr.net/npm/@pixiv/three-vrm-animation@3.2.0/lib/three-vrm-animation.module.min.js"
99
+ }
100
+ }
101
+ </script>
102
+ </head>
103
+ <body>
104
+ <div id="c"></div>
105
+ <div id="speakDot">
106
+ <div class="sdot"></div>
107
+ <div class="sdot"></div>
108
+ <div class="sdot"></div>
109
+ </div>
110
+ <div id="load">thinking…</div>
111
+ <button id="micBtn">🎙️</button>
112
+
113
+ <script type="module">
114
+ import * as THREE from 'three';
115
+ import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js';
116
+ import { VRMLoaderPlugin, VRMUtils } from '@pixiv/three-vrm';
117
+ import { VRMAnimationLoaderPlugin, createVRMAnimationClip } from '@pixiv/three-vrm-animation';
118
+ import { OrbitControls } from 'three/addons/controls/OrbitControls.js';
119
+
120
+ // Parse query params passed down by Gradio
121
+ const urlParams = new URLSearchParams(window.location.search);
122
+ const vrmPath = urlParams.get('vrm_path') || '';
123
+ const animationDir = urlParams.get('animation_dir') || '';
124
+
125
+ let availableVrmas = [];
126
+ try {
127
+ const animationsParam = urlParams.get('animations');
128
+ if (animationsParam) {
129
+ availableVrmas = JSON.parse(decodeURIComponent(animationsParam));
130
+ }
131
+ } catch(e) {
132
+ console.error("Error parsing animations parameter:", e);
133
+ }
134
+
135
+ // Setup Scene, Camera and Renderer
136
+ const container = document.getElementById('c');
137
+ const scene = new THREE.Scene();
138
+ scene.background = new THREE.Color(0xffe082);
139
+
140
+ const camera = new THREE.PerspectiveCamera(45, window.innerWidth / window.innerHeight, 0.1, 100);
141
+ camera.position.set(0, 1.4, 1.8);
142
+
143
+ let renderer;
144
+ try {
145
+ renderer = new THREE.WebGLRenderer({ antialias: true, alpha: true, powerPreference: 'high-performance' });
146
+ renderer.setSize(window.innerWidth, window.innerHeight);
147
+ renderer.setPixelRatio(Math.min(window.devicePixelRatio, 2));
148
+ renderer.outputColorSpace = THREE.SRGBColorSpace;
149
+ container.appendChild(renderer.domElement);
150
+ } catch (err) {
151
+ console.error("WebGL initialization failed:", err);
152
+ const warn = document.createElement('div');
153
+ warn.style.position = 'absolute';
154
+ warn.style.top = '50%';
155
+ warn.style.left = '50%';
156
+ warn.style.transform = 'translate(-50%, -50%)';
157
+ warn.style.color = '#ff4b4b';
158
+ warn.style.fontSize = '18px';
159
+ warn.style.textAlign = 'center';
160
+ warn.innerHTML = "<strong>WebGL Error</strong><br>Your browser does not support WebGL.";
161
+ document.body.appendChild(warn);
162
+ }
163
+
164
+ const orbit = new OrbitControls(camera, renderer.domElement);
165
+ orbit.target.set(0, 1.3, 0);
166
+ orbit.enableDamping = true;
167
+ orbit.update();
168
+
169
+ // Lighting
170
+ scene.add(new THREE.AmbientLight(0xfff4e0, 0.5));
171
+ const keyLight = new THREE.DirectionalLight(0xffffff, 1.2);
172
+ keyLight.position.set(2, 4, 3);
173
+ scene.add(keyLight);
174
+
175
+ let currentVrm = null;
176
+ let mixer = null;
177
+ let audioContext = null, analyser = null, dataArray = null;
178
+ let isSpeaking = false;
179
+ let blinkT = 0, nextBlink = 3, blinkV = 0;
180
+
181
+ let activeExpressions = { 'neutral': 1.0 };
182
+ let currentVrmaAction = null;
183
+
184
+ // GLTF Loader Setup
185
+ const loader = new GLTFLoader();
186
+ try {
187
+ loader.register(p => new VRMLoaderPlugin(p));
188
+ } catch (e) {
189
+ console.error("VRMLoaderPlugin registration failed:", e);
190
+ }
191
+ try {
192
+ loader.register(p => new VRMAnimationLoaderPlugin(p));
193
+ } catch (e) {
194
+ console.error("VRMAnimationLoaderPlugin registration failed:", e);
195
+ }
196
+
197
+ // VRMA Playback with robust fallback URLs
198
+ function playVrma(vrmaName) {
199
+ if (!currentVrm || !mixer) return;
200
+
201
+ let finalVrma = vrmaName;
202
+ if (!availableVrmas.includes(vrmaName)) {
203
+ if (availableVrmas.length > 0) {
204
+ finalVrma = availableVrmas[0];
205
+ }
206
+ }
207
+
208
+ const urls = [
209
+ `/file=animation/${finalVrma}`,
210
+ `/gradio_api/file=animation/${finalVrma}`,
211
+ `/file=${animationDir}/${finalVrma}`,
212
+ `/gradio_api/file=${animationDir}/${finalVrma}`
213
+ ];
214
+
215
+ const loadVrmaWithFallback = (vrmaUrls) => {
216
+ if (vrmaUrls.length === 0) {
217
+ console.warn("All VRMA animation URLs failed to load:", finalVrma);
218
+ return;
219
+ }
220
+ const url = vrmaUrls.shift();
221
+ loader.load(url, (gltf) => {
222
+ const vrmAnimations = gltf.userData.vrmAnimations;
223
+ if (vrmAnimations && vrmAnimations.length > 0) {
224
+ if (currentVrmaAction) {
225
+ currentVrmaAction.stop();
226
+ mixer.uncacheClip(currentVrmaAction.getClip());
227
+ }
228
+ const clip = createVRMAnimationClip(vrmAnimations[0], currentVrm);
229
+ currentVrmaAction = mixer.clipAction(clip);
230
+ currentVrmaAction.play();
231
+ console.log("Successfully loaded VRMA from:", url);
232
+ } else {
233
+ loadVrmaWithFallback(vrmaUrls);
234
+ }
235
+ }, undefined, (err) => {
236
+ loadVrmaWithFallback(vrmaUrls);
237
+ });
238
+ };
239
+
240
+ loadVrmaWithFallback(urls);
241
+ }
242
+
243
+ // Load VRM Model with safe-guards and fallbacks
244
+ const loadVrmWithFallback = (urls) => {
245
+ if (urls.length === 0) {
246
+ console.error("All VRM model URLs failed to load.");
247
+ return;
248
+ }
249
+ const url = urls.shift();
250
+ console.log("Attempting to load VRM model from:", url);
251
+ loader.load(url, (gltf) => {
252
+ const vrm = gltf.userData.vrm;
253
+ if (!vrm) {
254
+ console.error("GLTF loaded but userData.vrm is undefined at:", url);
255
+ loadVrmWithFallback(urls);
256
+ return;
257
+ }
258
+
259
+ if (typeof VRMUtils !== 'undefined' && VRMUtils.rotateVRM0) {
260
+ VRMUtils.rotateVRM0(vrm);
261
+ } else {
262
+ if (vrm.meta && vrm.meta.metaVersion === '0') {
263
+ vrm.scene.rotation.y = Math.PI;
264
+ }
265
+ }
266
+
267
+ scene.add(vrm.scene);
268
+ currentVrm = vrm;
269
+ mixer = new THREE.AnimationMixer(vrm.scene);
270
+
271
+ if (vrm.expressionManager) {
272
+ vrm.expressionManager.setValue('neutral', 1.0);
273
+ } else if (vrm.blendShapeProxy) {
274
+ vrm.blendShapeProxy.setValue('neutral', 1.0);
275
+ }
276
+
277
+ let head = null;
278
+ if (vrm.humanoid) {
279
+ if (typeof vrm.humanoid.getRawBoneNode === 'function') {
280
+ head = vrm.humanoid.getRawBoneNode('head');
281
+ } else if (typeof vrm.humanoid.getBoneNode === 'function') {
282
+ head = vrm.humanoid.getBoneNode('head');
283
+ }
284
+ }
285
+
286
+ if (head) {
287
+ const worldPos = new THREE.Vector3();
288
+ head.getWorldPosition(worldPos);
289
+ orbit.target.copy(worldPos);
290
+ orbit.update();
291
+ }
292
+
293
+ // Play starting animation
294
+ if (availableVrmas.length > 0) {
295
+ playVrma(availableVrmas[0]);
296
+ }
297
+ console.log("Successfully loaded VRM model from:", url);
298
+ }, undefined, (e) => {
299
+ console.warn("Failed to load VRM from:", url, e);
300
+ loadVrmWithFallback(urls);
301
+ });
302
+ };
303
+
304
+ const modelUrls = [
305
+ '/file=model/Ani.vrm',
306
+ '/gradio_api/file=model/Ani.vrm',
307
+ `/file=${vrmPath}`,
308
+ `/gradio_api/file=${vrmPath}`
309
+ ];
310
+ loadVrmWithFallback(modelUrls);
311
+
312
+ function initAudioSync(audioElement) {
313
+ if (!audioContext) {
314
+ audioContext = new (window.AudioContext || window.webkitAudioContext)();
315
+ analyser = audioContext.createAnalyser();
316
+ analyser.fftSize = 256;
317
+ dataArray = new Uint8Array(analyser.frequencyBinCount);
318
+ }
319
+ if (audioContext.state === 'suspended') audioContext.resume();
320
+
321
+ try {
322
+ const source = audioContext.createMediaElementSource(audioElement);
323
+ source.connect(analyser);
324
+ analyser.connect(audioContext.destination);
325
+ } catch(e) {
326
+ console.warn("Audio sync connection ignored (possibly already connected):", e);
327
+ }
328
+ }
329
+
330
+ // Animation Loop
331
+ const clock = new THREE.Clock();
332
+ function animate() {
333
+ requestAnimationFrame(animate);
334
+ const dt = Math.min(clock.getDelta(), 0.1);
335
+
336
+ if (currentVrm) {
337
+ if (mixer) mixer.update(dt);
338
+
339
+ // Set active facial expressions (happy, sad, etc.)
340
+ if (activeExpressions) {
341
+ for (const [key, val] of Object.entries(activeExpressions)) {
342
+ if (currentVrm.expressionManager) {
343
+ currentVrm.expressionManager.setValue(key, val);
344
+ } else if (currentVrm.blendShapeProxy) {
345
+ currentVrm.blendShapeProxy.setValue(key, val);
346
+ }
347
+ }
348
+ }
349
+
350
+ // Speaking / Lip Sync
351
+ if (isSpeaking && analyser) {
352
+ analyser.getByteFrequencyData(dataArray);
353
+ let sum = 0;
354
+ for (let i = 0; i < 32; i++) sum += dataArray[i];
355
+ const level = sum / 32 / 255;
356
+ const open = Math.min(level * 4.5, 1.2);
357
+ if (currentVrm.expressionManager) {
358
+ currentVrm.expressionManager.setValue('aa', open);
359
+ currentVrm.expressionManager.setValue('oh', open * 0.4);
360
+ currentVrm.expressionManager.setValue('ee', open * 0.2);
361
+ } else if (currentVrm.blendShapeProxy) {
362
+ currentVrm.blendShapeProxy.setValue('aa', open);
363
+ currentVrm.blendShapeProxy.setValue('oh', open * 0.4);
364
+ currentVrm.blendShapeProxy.setValue('ee', open * 0.2);
365
+ }
366
+ } else {
367
+ if (currentVrm.expressionManager) {
368
+ currentVrm.expressionManager.setValue('aa', 0);
369
+ currentVrm.expressionManager.setValue('oh', 0);
370
+ currentVrm.expressionManager.setValue('ee', 0);
371
+ } else if (currentVrm.blendShapeProxy) {
372
+ currentVrm.blendShapeProxy.setValue('aa', 0);
373
+ currentVrm.blendShapeProxy.setValue('oh', 0);
374
+ currentVrm.blendShapeProxy.setValue('ee', 0);
375
+ }
376
+ }
377
+
378
+ // Eye Blinking
379
+ const isEmotionActive = activeExpressions && (activeExpressions.happy > 0.5 || activeExpressions.surprised > 0.5 || activeExpressions.relaxed > 0.5);
380
+ if (!isSpeaking && !isEmotionActive) {
381
+ blinkT += dt;
382
+ if (blinkT > nextBlink) {
383
+ if (blinkV === 0) blinkV = 1;
384
+ if (blinkV === 1) {
385
+ let v = 0;
386
+ if (currentVrm.expressionManager) {
387
+ v = currentVrm.expressionManager.getValue('blink') || 0;
388
+ } else if (currentVrm.blendShapeProxy) {
389
+ v = currentVrm.blendShapeProxy.getValue('blink') || 0;
390
+ }
391
+ v += dt * 12;
392
+ if (v >= 1) { v = 1; blinkV = -1; }
393
+ if (currentVrm.expressionManager) {
394
+ currentVrm.expressionManager.setValue('blink', v);
395
+ } else if (currentVrm.blendShapeProxy) {
396
+ currentVrm.blendShapeProxy.setValue('blink', v);
397
+ }
398
+ } else {
399
+ let v = 1;
400
+ if (currentVrm.expressionManager) {
401
+ v = currentVrm.expressionManager.getValue('blink') || 1;
402
+ } else if (currentVrm.blendShapeProxy) {
403
+ v = currentVrm.blendShapeProxy.getValue('blink') || 1;
404
+ }
405
+ v -= dt * 12;
406
+ if (v <= 0) { v = 0; blinkV = 0; blinkT = 0; nextBlink = 2 + Math.random() * 5; }
407
+ if (currentVrm.expressionManager) {
408
+ currentVrm.expressionManager.setValue('blink', v);
409
+ } else if (currentVrm.blendShapeProxy) {
410
+ currentVrm.blendShapeProxy.setValue('blink', v);
411
+ }
412
+ }
413
+ }
414
+ } else {
415
+ if (currentVrm.expressionManager) {
416
+ currentVrm.expressionManager.setValue('blink', 0);
417
+ } else if (currentVrm.blendShapeProxy) {
418
+ currentVrm.blendShapeProxy.setValue('blink', 0);
419
+ }
420
+ }
421
+ currentVrm.update(dt);
422
+ }
423
+ orbit.update();
424
+ if (renderer) renderer.render(scene, camera);
425
+ }
426
+ animate();
427
+
428
+ // Listen for parent messages (from Gradio host)
429
+ window.addEventListener('message', (e) => {
430
+ if (e.data.type === 'vrm_update') {
431
+ const { audio_url, data } = e.data;
432
+ const audio = new Audio(audio_url);
433
+
434
+ initAudioSync(audio);
435
+ isSpeaking = true;
436
+ document.getElementById('speakDot').classList.add('on');
437
+ audio.play().catch(err => console.log("Audio play deferred or failed:", err));
438
+
439
+ audio.onended = () => {
440
+ isSpeaking = false;
441
+ document.getElementById('speakDot').classList.remove('on');
442
+ };
443
+
444
+ if (data.vrma) {
445
+ playVrma(data.vrma);
446
+ }
447
+
448
+ if (currentVrm && data.expressions) {
449
+ activeExpressions = {};
450
+ ['happy','sad','angry','surprised','relaxed','neutral'].forEach(ex => {
451
+ if (currentVrm.expressionManager) {
452
+ currentVrm.expressionManager.setValue(ex, 0);
453
+ } else if (currentVrm.blendShapeProxy) {
454
+ currentVrm.blendShapeProxy.setValue(ex, 0);
455
+ }
456
+ });
457
+ for (const [key, val] of Object.entries(data.expressions)) {
458
+ activeExpressions[key] = val;
459
+ }
460
+ if (Object.keys(activeExpressions).length === 0) {
461
+ activeExpressions['neutral'] = 1.0;
462
+ }
463
+ }
464
+ } else if (e.data.type === 'set_loading') {
465
+ const loadOverlay = document.getElementById('load');
466
+ if (e.data.val) {
467
+ loadOverlay.classList.add('on');
468
+ } else {
469
+ loadOverlay.classList.remove('on');
470
+ }
471
+ }
472
+ });
473
+
474
+ // Speech Recognition / Voice Mic Handler
475
+ const micBtn = document.getElementById('micBtn');
476
+ if ('webkitSpeechRecognition' in window || 'SpeechRecognition' in window) {
477
+ const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition;
478
+ const recognition = new SpeechRecognition();
479
+ recognition.continuous = false;
480
+ recognition.interimResults = false;
481
+ recognition.lang = 'en-US';
482
+
483
+ recognition.onstart = () => {
484
+ micBtn.style.background = 'rgba(255, 75, 75, 0.9)';
485
+ micBtn.style.color = '#fff';
486
+ const loadOverlay = document.getElementById('load');
487
+ loadOverlay.textContent = "Listening...";
488
+ loadOverlay.classList.add('on');
489
+ };
490
+ recognition.onend = () => {
491
+ micBtn.style.background = 'rgba(15, 18, 32, 0.92)';
492
+ micBtn.style.color = '#6cf';
493
+ const loadOverlay = document.getElementById('load');
494
+ loadOverlay.classList.remove('on');
495
+ loadOverlay.textContent = "thinking…";
496
+ };
497
+ recognition.onresult = (event) => {
498
+ const transcript = event.results[0][0].transcript;
499
+ // Dispatch text back to Gradio host parent window
500
+ window.parent.postMessage({
501
+ type: 'mic_transcript',
502
+ text: transcript
503
+ }, '*');
504
+ };
505
+ micBtn.addEventListener('click', () => {
506
+ try {
507
+ recognition.start();
508
+ } catch(e) {
509
+ recognition.stop();
510
+ }
511
+ });
512
+ } else {
513
+ micBtn.style.display = 'none';
514
+ }
515
+
516
+ window.addEventListener('resize', () => {
517
+ camera.aspect = window.innerWidth / window.innerHeight;
518
+ camera.updateProjectionMatrix();
519
+ if (renderer) renderer.setSize(window.innerWidth, window.innerHeight);
520
+ });
521
+ </script>
522
+ </body>
523
+ </html>