chenbhao Claude Big Pickle commited on
Commit
7ec132a
·
1 Parent(s): 762b1c8

feat: /friend aliveness system, motion controller fix, VAD tuning, high-pass filter

Browse files

- Remove redundant files (HistoryPanel, MoodIndicator, vmd-loader, vrm-ik-handler, web-file-picker, FriendScreenObserveTool)
- Add action parameter to friend_emotion for VRM animation playback
- Fix motion-controller: replace broken 'finished' event with frame-accurate completion detection and action queue
- Add aliveness system: breathing, postural sway, speech micro-movements, listening response, micro-expressions, enhanced blink
- Add Silero VAD threshold tuning and high-pass filter for low-frequency noise rejection
- Fix build scripts (vite build instead of tauri build)
- Add LipSync.isActive() for speech-driven animation blending

Co-Authored-By: Claude Big Pickle <noreply@anthropic.com>

package.json CHANGED
@@ -18,7 +18,8 @@
18
  "scripts": {
19
  "build": "bun run ./scripts/build.ts",
20
  "dev": "bun run ./scripts/dev.ts",
21
- "friend:build": "cd src/components/friend/frontend && npx tauri build"
 
22
  },
23
  "dependencies": {
24
  "@alcalzone/ansi-tokenize": "^0.3.0",
 
18
  "scripts": {
19
  "build": "bun run ./scripts/build.ts",
20
  "dev": "bun run ./scripts/dev.ts",
21
+ "friend:build": "cd src/components/friend/frontend && npx vite build",
22
+ "friend:dev": "cd src/components/friend/frontend && npx vite"
23
  },
24
  "dependencies": {
25
  "@alcalzone/ansi-tokenize": "^0.3.0",
src/components/friend/frontend/App.tsx CHANGED
@@ -1,27 +1,19 @@
1
  import { useEffect, useRef, useState, useCallback } from 'react'
2
  import { VRMScene } from './components/VRMScene'
3
- import type { VRMSceneHandle, TouchRegion } from './components/VRMScene'
4
  import { TextBubble } from './components/TextBubble'
5
  import type { OnVrmMessage } from './components/TextBubble'
6
  import { ChatInput } from './components/ChatInput'
7
  import { ResizeHandles } from './components/ResizeHandles'
8
  import { SettingsPanel } from './components/SettingsPanel'
9
  import { usePassThrough } from './hooks/usePassThrough'
10
- import { dancePresets } from './motion-controller'
11
  import { LipSync } from './lip-sync'
12
  import { FRIEND_API, bindScene } from './api'
13
- import { HistoryPanel } from './components/HistoryPanel'
14
- import { MoodIndicator } from './components/MoodIndicator'
15
- import { Menu, Pin, Move, RotateCcw, Rotate3D, EyeOff, Settings, Music, RefreshCw } from 'lucide-react'
16
 
17
  const DEFAULT_MODEL = '/friend/model1.vrm'
18
- // Module-level to survive any component remount
19
- let lastTouchMemoTime = 0
20
- const TOUCH_MEMO_COOLDOWN = 3_000
21
- let lastTouchChatTime = 0
22
- const TOUCH_CHAT_COOLDOWN = 60_000
23
 
24
- // 情绪 → 动作映射 (action names from motion-controller presets)
25
  const emotionActionMap: Record<string, string> = {
26
  think: 'scratchHead',
27
  question: 'point',
@@ -38,7 +30,6 @@ const emotionActionMap: Record<string, string> = {
38
  neutral: 'salute',
39
  }
40
 
41
-
42
  const btnStyle: React.CSSProperties = {
43
  width: 32,
44
  height: 32,
@@ -60,24 +51,16 @@ export default function App() {
60
  const [tracking, setTracking] = useState<'mouse' | 'camera'>('mouse')
61
  const [showText, setShowText] = useState(true)
62
  const [collapsed, setCollapsed] = useState(false)
63
-
64
  const [ttsEnabled, setTtsEnabled] = useState(true)
65
  const [modelPath, setModelPath] = useState(DEFAULT_MODEL)
66
  const [settingsOpen, setSettingsOpen] = useState(false)
67
- const [historyOpen, setHistoryOpen] = useState(false)
68
  const [hideUI, setHideUI] = useState(false)
69
  const [volume, setVolume] = useState(0.5)
70
  const [uiAlign, setUiAlign] = useState<'left' | 'right'>('right')
71
- const [dancing, setDancing] = useState(false)
72
- const [currentDance, setCurrentDance] = useState('jile')
73
- const [customDancePreset, setCustomDancePreset] = useState<import('./motion-controller').DancePreset | undefined>(undefined)
74
- const [hideMood, setHideMood] = useState(false)
75
- const [screenObserve, setScreenObserve] = useState(false)
76
- const [screenObserveInterval, setScreenObserveInterval] = useState(60)
77
  const [language, setLanguage] = useState<'zh' | 'en'>(() => navigator.language.startsWith('zh') ? 'zh' : 'en')
78
  const [sttProvider, setSttProvider] = useState<'browser' | 'groq' | 'anthropic' | 'local' | 'doubao'>('browser')
79
  const t = (zh: string, en: string) => language === 'en' ? en : zh
80
- usePassThrough(!settingsOpen && !historyOpen)
81
 
82
  // Load persisted settings on mount
83
  useEffect(() => {
@@ -89,18 +72,12 @@ export default function App() {
89
  if (s.showText !== undefined) setShowText(s.showText)
90
  if (s.hideUI !== undefined) setHideUI(s.hideUI)
91
  if (s.tracking) { setTracking(s.tracking); sceneRef.current?.setTrackingMode(s.tracking) }
92
- if (s.volume !== undefined) { setVolume(s.volume); LipSync.getInstance().setVolume(s.volume); sceneRef.current?.setBgmVolume(s.volume) }
93
  if (s.uiAlign) setUiAlign(s.uiAlign)
94
- if (s.hideMood !== undefined) setHideMood(s.hideMood)
95
- if (s.screenObserve !== undefined) setScreenObserve(s.screenObserve)
96
- if (s.screenObserveInterval !== undefined) setScreenObserveInterval(s.screenObserveInterval)
97
- if (s.currentDance) setCurrentDance(s.currentDance)
98
- if (s.customDancePreset) setCustomDancePreset(s.customDancePreset)
99
  if (s.sttProvider) setSttProvider(s.sttProvider)
100
  if (s.language) {
101
  setLanguage(s.language)
102
  } else {
103
- // No saved language — persist the detected system language to backend
104
  const detected = navigator.language.startsWith('zh') ? 'zh' : 'en'
105
  fetch(`${FRIEND_API}/settings`, {
106
  method: 'POST',
@@ -120,10 +97,6 @@ export default function App() {
120
  }).catch(() => {})
121
  }
122
 
123
- useEffect(() => {
124
- // open-settings Tauri event replaced by F4 keyboard shortcut (handled below)
125
- }, [])
126
-
127
  useEffect(() => {
128
  bindScene(sceneRef.current)
129
  return () => bindScene(null)
@@ -132,7 +105,6 @@ export default function App() {
132
  const handleVolumeChange = useCallback((v: number) => {
133
  setVolume(v)
134
  LipSync.getInstance().setVolume(v)
135
- sceneRef.current?.setBgmVolume(v)
136
  saveSettings({ volume: v })
137
  }, [])
138
 
@@ -144,22 +116,19 @@ export default function App() {
144
 
145
  const handleVrmMessage: OnVrmMessage = useCallback((msg) => {
146
  if (msg.emotion && sceneRef.current) {
147
- const action = emotionActionMap[msg.emotion]
148
  if (msg.text) {
149
- // 回复消息:表情和动作同时触发(文字出现1s后由TextBubble延迟调用)
150
  sceneRef.current.setEmotionWithReset(msg.emotion, msg.emotionDuration ?? 5000, msg.emotionIntensity)
151
  if (action) sceneRef.current.playAction(action)
152
  } else {
153
- // 思考阶段:hold 动作,10s 后自动 reset
154
  sceneRef.current.setEmotionWithReset(msg.emotion, msg.emotionDuration ?? 10000, msg.emotionIntensity)
155
  if (action) sceneRef.current.playAction(action, true)
156
  }
157
  }
158
  }, [])
159
 
160
- // ── Idle fidget: random emotion + action every ~60s when idle ──────────────
161
  const lastActivityRef = useRef(Date.now())
162
- // Reset idle timer whenever a VRM message arrives
163
  const originalHandleVrmMessage = handleVrmMessage
164
  const handleVrmMessageWithActivity: OnVrmMessage = useCallback((msg) => {
165
  lastActivityRef.current = Date.now()
@@ -171,84 +140,52 @@ export default function App() {
171
  'happy', 'sad', 'angry', 'surprised', 'think', 'awkward',
172
  'question', 'curious', 'neutral', 'love', 'flirty', 'greeting', 'relaxed',
173
  ]
174
- const allActions = [
175
- 'akimbo', 'playFingers', 'scratchHead', 'stretch',
176
- 'happy', 'angry', 'greeting', 'excited', 'shy',
177
- 'point', 'salute', 'angryPump',
 
 
 
 
 
 
 
 
 
 
 
 
 
178
  ]
179
- const IDLE_THRESHOLD_MS = 30_000
180
- const FIDGET_CHECK_MS = 15_000 // check every 15s, randomness inside
181
 
182
  const timer = setInterval(() => {
183
  const idleMs = Date.now() - lastActivityRef.current
184
  if (idleMs < IDLE_THRESHOLD_MS) return
185
- // 50% chance each check to avoid being too predictable
186
- if (Math.random() > 0.5) return
187
-
188
- const emotion = allEmotions[Math.floor(Math.random() * allEmotions.length)]
189
- const action = allActions[Math.floor(Math.random() * allActions.length)]
190
- const intensity = 0.4 + Math.random() * 0.4 // 0.4–0.8
191
- sceneRef.current?.setEmotionWithReset(emotion, 3000 + Math.random() * 2000, intensity)
192
- sceneRef.current?.playAction(action)
193
- lastActivityRef.current = Date.now() // reset so we don't spam
194
- }, FIDGET_CHECK_MS)
195
-
196
- return () => clearInterval(timer)
197
- }, [])
198
-
199
- // ── Dancing mood boost: +1 every 30s ─────────
200
- useEffect(() => {
201
- if (!dancing) return
202
- const timer = setInterval(() => {
203
- fetch(`${FRIEND_API}/mood/adjust`, {
204
- method: 'POST',
205
- headers: { 'Content-Type': 'application/json' },
206
- body: JSON.stringify({ delta: 1, max: 90 }),
207
- }).catch(() => {})
208
- }, 30_000)
209
- return () => clearInterval(timer)
210
- }, [dancing])
211
 
212
- // ── Screen observation: capture desktop & send to LLM periodically ─────────
213
- useEffect(() => {
214
- if (!screenObserve) return
215
- const intervalMs = screenObserveInterval * 1000
216
-
217
- const doObserve = () => {
218
- fetch(`${FRIEND_API}/screen/observe`, { method: 'POST' })
219
- .catch(() => {})
220
- }
221
 
222
- const timer = setInterval(doObserve, intervalMs)
223
- // Also fire once immediately on enable
224
- doObserve()
 
 
 
 
 
 
 
225
 
226
  return () => clearInterval(timer)
227
- }, [screenObserve, screenObserveInterval])
228
-
229
- // Capture VRM screenshot and upload to server (debounced to avoid duplicates)
230
- const screenshotTimer = useRef<ReturnType<typeof setTimeout> | null>(null)
231
- const uploadVrmScreenshot = useCallback(() => {
232
- if (screenshotTimer.current) clearTimeout(screenshotTimer.current)
233
- screenshotTimer.current = setTimeout(() => {
234
- screenshotTimer.current = null
235
- const dataUrl = sceneRef.current?.captureScreenshot()
236
- if (!dataUrl) return
237
- fetch(`${FRIEND_API}/persona/screenshot`, {
238
- method: 'POST',
239
- headers: { 'Content-Type': 'application/json' },
240
- body: JSON.stringify({ image: dataUrl }),
241
- }).catch(() => {})
242
- }, 500)
243
  }, [])
244
 
245
- const clearContext = async () => {
246
- try {
247
- await fetch(`${FRIEND_API}/context/clear`, { method: 'POST' })
248
- } catch { /* ignore */ }
249
- }
250
-
251
- // 全局快捷键: Tab 展开/折叠菜单
252
  useEffect(() => {
253
  const onKeyDown = (e: KeyboardEvent) => {
254
  if (e.target instanceof HTMLInputElement || e.target instanceof HTMLTextAreaElement) return
@@ -269,91 +206,7 @@ export default function App() {
269
  return () => window.removeEventListener('keydown', onKeyDown)
270
  }, [])
271
 
272
- // ── Touch interaction: immediate reaction + verbal response ──────────
273
- // Each region has multiple possible reactions, randomly selected (from lobe-vidol female defaults)
274
- const touchReactions: Record<TouchRegion, { emotion: string; action?: string }[]> = {
275
- head: [
276
- { emotion: 'relaxed', action: 'happy' },
277
- { emotion: 'relaxed', action: 'shy' },
278
- { emotion: 'relaxed', action: 'happy' },
279
- { emotion: 'relaxed', action: 'shy' },
280
- { emotion: 'angry', action: 'angryPump' },
281
- { emotion: 'relaxed', action: 'excited' },
282
- ],
283
- arm: [
284
- { emotion: 'surprised', action: 'excited' },
285
- { emotion: 'happy', action: 'happy' },
286
- { emotion: 'relaxed', action: 'greeting' },
287
- { emotion: 'relaxed', action: 'salute' },
288
- { emotion: 'relaxed', action: 'akimbo' },
289
- ],
290
- chest: [
291
- { emotion: 'angry', action: 'angryPump' },
292
- { emotion: 'angry', action: 'angry' },
293
- { emotion: 'angry', action: 'point' },
294
- ],
295
- belly: [
296
- { emotion: 'angry', action: 'angryPump' },
297
- { emotion: 'angry', action: 'angry' },
298
- { emotion: 'awkward', action: 'playFingers' },
299
- { emotion: 'angry', action: 'akimbo' },
300
- ],
301
- buttocks: [
302
- { emotion: 'angry', action: 'angryPump' },
303
- { emotion: 'angry', action: 'point' },
304
- { emotion: 'sad', action: 'shy' },
305
- ],
306
- leg: [
307
- { emotion: 'sad', action: 'shy' },
308
- { emotion: 'angry', action: 'angry' },
309
- { emotion: 'angry', action: 'point' },
310
- { emotion: 'angry', action: 'angryPump' },
311
- { emotion: 'awkward', action: 'playFingers' },
312
- ],
313
- }
314
-
315
- const regionLabels: Record<TouchRegion, string> = language === 'en'
316
- ? { head: 'head', arm: 'arm', chest: 'chest', belly: 'belly', buttocks: 'buttocks', leg: 'leg' }
317
- : { head: '头', arm: '手臂', chest: '胸', belly: '肚子', buttocks: '屁股', leg: '腿' }
318
-
319
- const handleTouch = useCallback((region: TouchRegion) => {
320
- const reactions = touchReactions[region]
321
- if (!reactions?.length) return
322
- const visual = reactions[Math.floor(Math.random() * reactions.length)]
323
-
324
- // Immediate visual feedback (always)
325
- lastActivityRef.current = Date.now()
326
- sceneRef.current?.setEmotionWithReset(visual.emotion, 3000, 0.8)
327
- if (visual.action) sceneRef.current?.playAction(visual.action)
328
-
329
- const now = Date.now()
330
- const prompt = language === 'en'
331
- ? `[The user touched your ${regionLabels[region]}]`
332
- : `[用户摸了摸你的${regionLabels[region]}]`
333
-
334
- // Always write to session history (3s cooldown)
335
- if (now - lastTouchMemoTime > TOUCH_MEMO_COOLDOWN) {
336
- lastTouchMemoTime = now
337
- fetch(`${FRIEND_API}/session/memo`, {
338
- method: 'POST',
339
- headers: { 'Content-Type': 'application/json' },
340
- body: JSON.stringify({ text: prompt }),
341
- }).catch(() => {})
342
- }
343
-
344
- // Send to backend for verbal response (60s cooldown, 50% chance)
345
- if (now - lastTouchChatTime > TOUCH_CHAT_COOLDOWN && Math.random() < 0.5) {
346
- lastTouchChatTime = now
347
- fetch(`${FRIEND_API}/touch`, {
348
- method: 'POST',
349
- headers: { 'Content-Type': 'application/json' },
350
- body: JSON.stringify({ region, prompt }),
351
- }).catch(() => {})
352
- }
353
- }, [])
354
-
355
  const togglePin = async () => {
356
- // Window pinning not available in web mode
357
  setPinned((v) => !v)
358
  }
359
 
@@ -368,20 +221,14 @@ export default function App() {
368
  }}
369
  >
370
  <ResizeHandles />
371
- <VRMScene ref={sceneRef} modelPath={modelPath} onTouch={handleTouch} onModelLoaded={uploadVrmScreenshot} />
372
- {!hideMood && <MoodIndicator uiAlign={uiAlign} />}
373
  <TextBubble onMessage={handleVrmMessageWithActivity} enabled={showText} ttsEnabled={ttsEnabled} />
374
- {!hideUI && <ChatInput uiAlign={uiAlign} onHistoryOpen={() => setHistoryOpen(true)} onNewSession={clearContext} language={language} sttProvider={sttProvider} />}
375
- <HistoryPanel
376
- visible={historyOpen}
377
- onClose={() => setHistoryOpen(false)}
378
- language={language}
379
- />
380
  <SettingsPanel
381
  visible={settingsOpen}
382
  onClose={() => setSettingsOpen(false)}
383
  currentModel={modelPath}
384
- onModelChange={(m) => { setModelPath(m); setDancing(false); saveSettings({ modelPath: m }) }}
385
  hideUI={hideUI}
386
  onHideUIChange={(v) => { setHideUI(v); saveSettings({ hideUI: v }) }}
387
  showText={showText}
@@ -394,23 +241,10 @@ export default function App() {
394
  onVolumeChange={handleVolumeChange}
395
  uiAlign={uiAlign}
396
  onUiAlignChange={(v) => { setUiAlign(v); saveSettings({ uiAlign: v }) }}
397
- hideMood={hideMood}
398
- onHideMoodChange={(v) => { setHideMood(v); saveSettings({ hideMood: v }) }}
399
- screenObserve={screenObserve}
400
- onScreenObserveChange={(v) => { setScreenObserve(v); saveSettings({ screenObserve: v }) }}
401
- screenObserveInterval={screenObserveInterval}
402
- onScreenObserveIntervalChange={(v) => { setScreenObserveInterval(v); saveSettings({ screenObserveInterval: v }) }}
403
- captureVrmScreenshot={() => sceneRef.current?.captureScreenshot() ?? null}
404
  language={language}
405
  onLanguageChange={(v) => { setLanguage(v); saveSettings({ language: v }) }}
406
  sttProvider={sttProvider}
407
  onSttProviderChange={(v) => { setSttProvider(v); saveSettings({ sttProvider: v }) }}
408
- currentDance={currentDance}
409
- onDanceChange={(id, preset) => {
410
- setCurrentDance(id)
411
- setCustomDancePreset(preset)
412
- saveSettings({ currentDance: id, customDancePreset: preset })
413
- }}
414
  />
415
  {!hideUI && <div
416
  style={{
@@ -506,37 +340,6 @@ export default function App() {
506
  >
507
  <Rotate3D size={16} />
508
  </button>
509
- <button
510
- onClick={() => {
511
- if (dancing) {
512
- sceneRef.current?.reset()
513
- setDancing(false)
514
- } else {
515
- const label = currentDance.startsWith('custom:') && customDancePreset
516
- ? customDancePreset.label
517
- : dancePresets[currentDance]?.label ?? currentDance
518
- if (currentDance.startsWith('custom:') && customDancePreset) {
519
- sceneRef.current?.playDance(customDancePreset)
520
- } else {
521
- sceneRef.current?.playDance(currentDance)
522
- }
523
- setDancing(true)
524
- // Record dance event in session history (no LLM reply)
525
- fetch(`${FRIEND_API}/session/memo`, {
526
- method: 'POST',
527
- headers: { 'Content-Type': 'application/json' },
528
- body: JSON.stringify({ text: `[用户邀请你跳了一支舞:${label}]` }),
529
- }).catch(() => {})
530
- }
531
- }}
532
- style={{
533
- ...btnStyle,
534
- ...(dancing ? { background: 'rgba(34, 197, 94, 0.6)' } : {}),
535
- }}
536
- title={dancing ? t('停止跳舞', 'Stop Dance') : t('跳舞', 'Dance')}
537
- >
538
- <Music size={16} />
539
- </button>
540
  </>}
541
  </div>}
542
  </div>
 
1
  import { useEffect, useRef, useState, useCallback } from 'react'
2
  import { VRMScene } from './components/VRMScene'
3
+ import type { VRMSceneHandle } from './components/VRMScene'
4
  import { TextBubble } from './components/TextBubble'
5
  import type { OnVrmMessage } from './components/TextBubble'
6
  import { ChatInput } from './components/ChatInput'
7
  import { ResizeHandles } from './components/ResizeHandles'
8
  import { SettingsPanel } from './components/SettingsPanel'
9
  import { usePassThrough } from './hooks/usePassThrough'
 
10
  import { LipSync } from './lip-sync'
11
  import { FRIEND_API, bindScene } from './api'
12
+ import { Menu, Move, Rotate3D, EyeOff, Settings, RefreshCw, Pin } from 'lucide-react'
 
 
13
 
14
  const DEFAULT_MODEL = '/friend/model1.vrm'
 
 
 
 
 
15
 
16
+ // 情绪 → 动作映射
17
  const emotionActionMap: Record<string, string> = {
18
  think: 'scratchHead',
19
  question: 'point',
 
30
  neutral: 'salute',
31
  }
32
 
 
33
  const btnStyle: React.CSSProperties = {
34
  width: 32,
35
  height: 32,
 
51
  const [tracking, setTracking] = useState<'mouse' | 'camera'>('mouse')
52
  const [showText, setShowText] = useState(true)
53
  const [collapsed, setCollapsed] = useState(false)
 
54
  const [ttsEnabled, setTtsEnabled] = useState(true)
55
  const [modelPath, setModelPath] = useState(DEFAULT_MODEL)
56
  const [settingsOpen, setSettingsOpen] = useState(false)
 
57
  const [hideUI, setHideUI] = useState(false)
58
  const [volume, setVolume] = useState(0.5)
59
  const [uiAlign, setUiAlign] = useState<'left' | 'right'>('right')
 
 
 
 
 
 
60
  const [language, setLanguage] = useState<'zh' | 'en'>(() => navigator.language.startsWith('zh') ? 'zh' : 'en')
61
  const [sttProvider, setSttProvider] = useState<'browser' | 'groq' | 'anthropic' | 'local' | 'doubao'>('browser')
62
  const t = (zh: string, en: string) => language === 'en' ? en : zh
63
+ usePassThrough(!settingsOpen)
64
 
65
  // Load persisted settings on mount
66
  useEffect(() => {
 
72
  if (s.showText !== undefined) setShowText(s.showText)
73
  if (s.hideUI !== undefined) setHideUI(s.hideUI)
74
  if (s.tracking) { setTracking(s.tracking); sceneRef.current?.setTrackingMode(s.tracking) }
75
+ if (s.volume !== undefined) { setVolume(s.volume); LipSync.getInstance().setVolume(s.volume) }
76
  if (s.uiAlign) setUiAlign(s.uiAlign)
 
 
 
 
 
77
  if (s.sttProvider) setSttProvider(s.sttProvider)
78
  if (s.language) {
79
  setLanguage(s.language)
80
  } else {
 
81
  const detected = navigator.language.startsWith('zh') ? 'zh' : 'en'
82
  fetch(`${FRIEND_API}/settings`, {
83
  method: 'POST',
 
97
  }).catch(() => {})
98
  }
99
 
 
 
 
 
100
  useEffect(() => {
101
  bindScene(sceneRef.current)
102
  return () => bindScene(null)
 
105
  const handleVolumeChange = useCallback((v: number) => {
106
  setVolume(v)
107
  LipSync.getInstance().setVolume(v)
 
108
  saveSettings({ volume: v })
109
  }, [])
110
 
 
116
 
117
  const handleVrmMessage: OnVrmMessage = useCallback((msg) => {
118
  if (msg.emotion && sceneRef.current) {
119
+ const action = msg.action || emotionActionMap[msg.emotion]
120
  if (msg.text) {
 
121
  sceneRef.current.setEmotionWithReset(msg.emotion, msg.emotionDuration ?? 5000, msg.emotionIntensity)
122
  if (action) sceneRef.current.playAction(action)
123
  } else {
 
124
  sceneRef.current.setEmotionWithReset(msg.emotion, msg.emotionDuration ?? 10000, msg.emotionIntensity)
125
  if (action) sceneRef.current.playAction(action, true)
126
  }
127
  }
128
  }, [])
129
 
130
+ // ── Idle fidget: natural micro-movements when no activity ─────────────────
131
  const lastActivityRef = useRef(Date.now())
 
132
  const originalHandleVrmMessage = handleVrmMessage
133
  const handleVrmMessageWithActivity: OnVrmMessage = useCallback((msg) => {
134
  lastActivityRef.current = Date.now()
 
140
  'happy', 'sad', 'angry', 'surprised', 'think', 'awkward',
141
  'question', 'curious', 'neutral', 'love', 'flirty', 'greeting', 'relaxed',
142
  ]
143
+ const idleTimers: Array<{ emotion?: string; action?: string; minIdle: number }> = [
144
+ // Emotion-only (subtle facial changes, no body action)
145
+ { emotion: 'think', minIdle: 8 },
146
+ { emotion: 'curious', minIdle: 12 },
147
+ { emotion: 'relaxed', minIdle: 15 },
148
+ { emotion: 'happy', minIdle: 20 },
149
+ // Small body actions (no strong emotion)
150
+ { action: 'stretch', minIdle: 25 },
151
+ { action: 'scratchHead', minIdle: 10 },
152
+ { action: 'playFingers', minIdle: 15 },
153
+ { action: 'akimbo', minIdle: 30 },
154
+ // Combined emotion + action
155
+ { emotion: 'think', action: 'scratchHead', minIdle: 18 },
156
+ { emotion: 'relaxed', action: 'akimbo', minIdle: 35 },
157
+ { emotion: 'curious', action: 'point', minIdle: 22 },
158
+ { emotion: 'happy', action: 'shy', minIdle: 28 },
159
+ { emotion: 'surprised', action: 'excited', minIdle: 40 },
160
  ]
161
+ const IDLE_THRESHOLD_MS = 15_000
162
+ const FIDGET_CHECK_MS = 8_000
163
 
164
  const timer = setInterval(() => {
165
  const idleMs = Date.now() - lastActivityRef.current
166
  if (idleMs < IDLE_THRESHOLD_MS) return
167
+ if (Math.random() > 0.55) return
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
168
 
169
+ // Pick a weighted-random fidget based on how long we've been idle
170
+ const candidates = idleTimers.filter((t) => idleMs >= t.minIdle * 1000)
171
+ if (candidates.length === 0) return
172
+ const pick = candidates[Math.floor(Math.random() * candidates.length)]
 
 
 
 
 
173
 
174
+ if (pick.emotion) {
175
+ const intensity = 0.3 + Math.random() * 0.5
176
+ const duration = 2000 + Math.random() * 3000
177
+ sceneRef.current?.setEmotionWithReset(pick.emotion, duration, intensity)
178
+ }
179
+ if (pick.action) {
180
+ sceneRef.current?.playAction(pick.action)
181
+ }
182
+ lastActivityRef.current = Date.now()
183
+ }, FIDGET_CHECK_MS)
184
 
185
  return () => clearInterval(timer)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
186
  }, [])
187
 
188
+ // 全局快捷键
 
 
 
 
 
 
189
  useEffect(() => {
190
  const onKeyDown = (e: KeyboardEvent) => {
191
  if (e.target instanceof HTMLInputElement || e.target instanceof HTMLTextAreaElement) return
 
206
  return () => window.removeEventListener('keydown', onKeyDown)
207
  }, [])
208
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
209
  const togglePin = async () => {
 
210
  setPinned((v) => !v)
211
  }
212
 
 
221
  }}
222
  >
223
  <ResizeHandles />
224
+ <VRMScene ref={sceneRef} modelPath={modelPath} />
 
225
  <TextBubble onMessage={handleVrmMessageWithActivity} enabled={showText} ttsEnabled={ttsEnabled} />
226
+ {!hideUI && <ChatInput uiAlign={uiAlign} language={language} sttProvider={sttProvider} />}
 
 
 
 
 
227
  <SettingsPanel
228
  visible={settingsOpen}
229
  onClose={() => setSettingsOpen(false)}
230
  currentModel={modelPath}
231
+ onModelChange={(m) => { setModelPath(m); saveSettings({ modelPath: m }) }}
232
  hideUI={hideUI}
233
  onHideUIChange={(v) => { setHideUI(v); saveSettings({ hideUI: v }) }}
234
  showText={showText}
 
241
  onVolumeChange={handleVolumeChange}
242
  uiAlign={uiAlign}
243
  onUiAlignChange={(v) => { setUiAlign(v); saveSettings({ uiAlign: v }) }}
 
 
 
 
 
 
 
244
  language={language}
245
  onLanguageChange={(v) => { setLanguage(v); saveSettings({ language: v }) }}
246
  sttProvider={sttProvider}
247
  onSttProviderChange={(v) => { setSttProvider(v); saveSettings({ sttProvider: v }) }}
 
 
 
 
 
 
248
  />
249
  {!hideUI && <div
250
  style={{
 
340
  >
341
  <Rotate3D size={16} />
342
  </button>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
343
  </>}
344
  </div>}
345
  </div>
src/components/friend/frontend/components/ChatInput.tsx CHANGED
@@ -1,198 +1,72 @@
1
  import { useState, useRef, useCallback, useEffect } from 'react'
2
- import { MessageCircle, Send, Loader, Mic, ChevronDown, History, SquarePen, Plus, Phone, PhoneOff } from 'lucide-react'
3
  import { FRIEND_API } from '../api'
4
  import { useServerStt, type SttProvider } from '../hooks/useServerStt'
5
 
6
- // Keyframes (claw-input-slide-up, claw-input-slide-down, claw-pulse) are in index.html <style>
7
-
8
- export function ChatInput({ visible = true, onActiveChange, uiAlign = 'right', onHistoryOpen, onNewSession, language = 'zh', sttProvider = 'browser' }: { visible?: boolean; onActiveChange?: (hasText: boolean) => void; uiAlign?: 'left' | 'right'; onHistoryOpen?: () => void; onNewSession?: () => void; language?: 'zh' | 'en'; sttProvider?: SttProvider }) {
9
  const t = (zh: string, en: string) => language === 'en' ? en : zh
10
- const [open, setOpen] = useState(false)
11
  const [text, setText] = useState('')
12
- const [sending, setSending] = useState(false)
13
  const [recording, setRecording] = useState(false)
14
- const [closing, setClosing] = useState(false)
15
- const [menuOpen, setMenuOpen] = useState(false)
16
- const [voiceCallActive, setVoiceCallActive] = useState(false)
17
- const inputRef = useRef<HTMLInputElement>(null)
18
  const voiceCallActiveRef = useRef(false)
19
- const endVoiceCallRef = useRef<() => void>(() => {})
20
  const serverStt = useServerStt()
21
-
22
- const closeBar = useCallback(() => {
23
- if (closing) return
24
- setMenuOpen(false)
25
- setClosing(true)
26
- setTimeout(() => { setOpen(false); setClosing(false) }, 220)
27
- }, [closing])
28
-
29
- const send = useCallback(async (msg?: string) => {
30
- const finalMsg = (msg ?? text).trim()
31
- if (!finalMsg || sending) return
32
- setSending(true)
33
- setText('')
34
- onActiveChange?.(false)
35
- try {
36
- await fetch(`${FRIEND_API}/chat`, {
37
- method: 'POST',
38
- headers: { 'Content-Type': 'application/json' },
39
- body: JSON.stringify({ message: finalMsg }),
40
- })
41
- } catch (err) {
42
- console.error('Failed to send message:', err)
43
- } finally {
44
- setSending(false)
45
- }
46
- }, [text, sending])
47
-
48
- const startRecording = useCallback(async () => {
49
- // All voice capture is server-side via cpal (no getUserMedia needed)
50
- try {
51
- await serverStt.startPushToTalk(sttProvider, language === 'en' ? 'en' : 'zh')
52
- setRecording(true)
53
- setOpen(true)
54
- } catch (err) {
55
- console.error('Server STT start error:', err)
56
- setRecording(false)
57
- }
58
- }, [sttProvider, serverStt, language])
59
-
60
- const stopRecording = useCallback(async () => {
61
- if (sttProvider === 'browser') {
62
- // Browser STT may not be available, just stop
63
- setRecording(false)
64
- return
65
- }
66
- setRecording(false)
67
- try {
68
- const text = await serverStt.stopPushToTalk()
69
- setText(text)
70
- onActiveChange?.(text.length > 0)
71
- } catch (err) {
72
- console.error('Server STT stop error:', err)
73
- }
74
- }, [serverStt, sttProvider, onActiveChange])
75
-
76
- const handleMouseLeave = useCallback(() => {
77
- if (recording) stopRecording()
78
- }, [recording, stopRecording])
79
-
80
- // --- Voice Call: delayed TTS interrupt ---
81
- const interruptTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null)
82
- const scheduleInterrupt = useCallback(() => {
83
- if (interruptTimerRef.current) return
84
- interruptTimerRef.current = setTimeout(() => {
85
- interruptTimerRef.current = null
86
- ;(window as any).__clawInterruptAudio?.()
87
- }, 1000)
88
- }, [])
89
- const cancelInterrupt = useCallback(() => {
90
- if (interruptTimerRef.current) { clearTimeout(interruptTimerRef.current); interruptTimerRef.current = null }
91
- }, [])
92
-
93
- // Voice call display timer — clear text after 3s
94
  const displayTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null)
95
 
96
- // --- Voice Call: start (browser VAD with @ricky0123/vad-web) ---
97
  const startVoiceCall = useCallback(async () => {
98
  setVoiceCallActive(true)
99
  voiceCallActiveRef.current = true
100
  setText('')
101
- setOpen(true)
102
 
103
  serverStt.startStreaming(
104
  sttProvider,
105
- (text, _isFinal) => {
106
- // Server-side transcribeAudioSegment() already calls sendText()
107
- // to enqueue the message into the CLI conversation.
108
- // We only need to display the transcript in the input bar briefly.
109
- if (text.trim()) {
110
- setText(text)
111
- onActiveChange?.(true)
112
- // Auto-clear after 3s
113
  if (displayTimerRef.current) clearTimeout(displayTimerRef.current)
114
  displayTimerRef.current = setTimeout(() => {
115
  if (voiceCallActiveRef.current) {
116
  setText('')
117
- onActiveChange?.(false)
118
  }
119
  }, 3000)
120
  }
121
  },
122
  (err) => {
123
- console.error('Browser VAD error:', err)
124
  if (voiceCallActiveRef.current) {
125
- setText(`语音启动失败: ${err}`)
126
  setTimeout(() => {
127
  setText('')
128
- endVoiceCallRef.current()
129
  }, 3000)
130
  }
131
  },
132
  language === 'en' ? 'en' : 'zh',
133
  )
134
  setRecording(true)
135
- }, [sttProvider, serverStt, language, onActiveChange])
136
 
137
- // --- Voice Call: end ---
138
  const endVoiceCall = useCallback(async () => {
139
- if (!voiceCallActiveRef.current) return // guard against double-invocation
140
  voiceCallActiveRef.current = false
141
  setVoiceCallActive(false)
142
  setRecording(false)
143
-
144
- // Stop VAD and clean up
145
  const finalText = await serverStt.stopStreaming()
146
-
147
- // Send any remaining transcript (no-op if empty — segments were already
148
- // sent to CLI by transcribeAudioSegment() during the call).
149
  if (finalText.trim()) {
150
- send(finalText)
 
 
 
 
 
151
  }
152
-
153
- cancelInterrupt()
154
-
155
  setText('')
156
- setOpen(false)
157
- onActiveChange?.(false)
158
- }, [onActiveChange, cancelInterrupt, serverStt, send])
159
-
160
- endVoiceCallRef.current = endVoiceCall
161
-
162
- const handleKeyDown = (e: React.KeyboardEvent) => {
163
- if (e.key === 'Enter' && !e.shiftKey) {
164
- e.preventDefault()
165
- send()
166
- }
167
- if (e.key === 'Escape') {
168
- closeBar()
169
- }
170
- if ((e.key === 'Delete') || (e.key === 'd' && e.ctrlKey)) {
171
- e.preventDefault()
172
- setText('')
173
- onActiveChange?.(false)
174
- }
175
- }
176
 
177
  // Global keyboard shortcuts
178
  useEffect(() => {
179
  const onGlobalKeyDown = (e: KeyboardEvent) => {
180
- if (!visible) return
181
- const inInput = e.target instanceof HTMLInputElement || e.target instanceof HTMLTextAreaElement
182
-
183
- if (e.key === 'Enter' && !e.shiftKey && !inInput) {
184
- e.preventDefault()
185
- if (!open) {
186
- setMenuOpen(false)
187
- setOpen(true)
188
- }
189
- setTimeout(() => inputRef.current?.focus(), 50)
190
- }
191
-
192
- if (e.key === 'Escape' && open && !inInput) {
193
- e.preventDefault()
194
- closeBar()
195
- }
196
 
197
  if (e.key === 'F2') {
198
  e.preventDefault()
@@ -205,58 +79,25 @@ export function ChatInput({ visible = true, onActiveChange, uiAlign = 'right', o
205
  }
206
  window.addEventListener('keydown', onGlobalKeyDown)
207
  return () => window.removeEventListener('keydown', onGlobalKeyDown)
208
- }, [open, visible, voiceCallActive, closeBar, startVoiceCall, endVoiceCall])
209
 
210
- if (!visible) return null
 
 
 
 
211
 
212
- if (!open) {
213
- return (
214
- <div style={{ position: 'absolute', bottom: 8, ...(uiAlign === 'left' ? { left: 12 } : { right: 12 }), zIndex: 300, pointerEvents: 'auto', display: 'flex', flexDirection: 'column', gap: 4, alignItems: 'center' }}>
215
- <button
216
- onClick={startVoiceCall}
217
- style={fabStyle}
218
- title={t('语音通话 (F2)', 'Voice Call (F2)')}
219
- >
220
- <Phone size={16} />
221
- </button>
222
- <button
223
- onClick={() => {
224
- setMenuOpen(false)
225
- setOpen(true)
226
- setTimeout(() => inputRef.current?.focus(), 50)
227
- }}
228
- style={fabStyle}
229
- title={t('发送消息 (Enter)', 'Send Message (Enter)')}
230
- >
231
- <MessageCircle size={16} />
232
- </button>
233
- </div>
234
- )
235
- }
236
 
237
  if (voiceCallActive) {
238
  return (
239
  <div style={barStyle}>
240
  <div style={{ flex: 1, position: 'relative' }}>
241
- <button
242
- onClick={() => setMenuOpen((v) => !v)}
243
- style={{ ...inlineBtnLeft, color: menuOpen ? 'rgba(100, 160, 255, 0.9)' : 'rgba(255, 255, 255, 0.45)' }}
244
- title={t('更多', 'More')}
245
- >
246
- <Plus size={22} />
247
- </button>
248
- {menuOpen && (
249
- <div style={popupMenuStyle}>
250
- <button onClick={() => { setMenuOpen(false); onHistoryOpen?.() }} style={popupItemStyle}>
251
- <History size={14} />
252
- <span>{t('对话历史', 'History')}</span>
253
- </button>
254
- <button onClick={() => { setMenuOpen(false); onNewSession?.() }} style={popupItemStyle}>
255
- <SquarePen size={14} />
256
- <span>{t('新会话', 'New Chat')}</span>
257
- </button>
258
- </div>
259
- )}
260
  <div
261
  style={{
262
  ...inputStyle,
@@ -274,8 +115,15 @@ export function ChatInput({ visible = true, onActiveChange, uiAlign = 'right', o
274
  </div>
275
  <span
276
  style={{
277
- ...inlineBtnRight,
 
 
278
  right: 44,
 
 
 
 
 
279
  color: recording ? 'rgba(255, 80, 80, 0.9)' : 'rgba(80, 200, 120, 0.9)',
280
  animation: recording ? 'claw-pulse 0.8s ease-in-out infinite' : 'none',
281
  transition: 'color 0.2s',
@@ -286,7 +134,23 @@ export function ChatInput({ visible = true, onActiveChange, uiAlign = 'right', o
286
  </span>
287
  <button
288
  onClick={endVoiceCall}
289
- style={{ ...inlineBtnRight, right: 12, color: 'rgba(255, 80, 80, 0.9)' }}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
290
  title={t('挂断 (F2)', 'Hang up (F2)')}
291
  >
292
  <PhoneOff size={22} />
@@ -297,89 +161,33 @@ export function ChatInput({ visible = true, onActiveChange, uiAlign = 'right', o
297
  }
298
 
299
  return (
300
- <>
301
- <div
302
- key={closing ? 'closing' : 'open'}
303
- style={{
304
- ...barStyle,
305
- animation: closing
306
- ? 'claw-input-slide-down 0.2s ease-in forwards'
307
- : 'claw-input-slide-up 0.25s ease-out both',
308
- }}
309
- >
310
- <div style={{ flex: 1, position: 'relative' }}>
311
- <button
312
- onClick={() => setMenuOpen((v) => !v)}
313
- style={{ ...inlineBtnLeft, color: menuOpen ? 'rgba(100, 160, 255, 0.9)' : 'rgba(255, 255, 255, 0.45)' }}
314
- title={t('更多', 'More')}
315
- >
316
- <Plus size={22} />
317
- </button>
318
- {menuOpen && (
319
- <div style={popupMenuStyle}>
320
- <button onClick={() => { setMenuOpen(false); onHistoryOpen?.() }} style={popupItemStyle}>
321
- <History size={14} />
322
- <span>{t('对话历史', 'History')}</span>
323
- </button>
324
- <button onClick={() => { setMenuOpen(false); onNewSession?.() }} style={popupItemStyle}>
325
- <SquarePen size={14} />
326
- <span>{t('新会话', 'New Chat')}</span>
327
- </button>
328
- </div>
329
- )}
330
- <input
331
- ref={inputRef}
332
- value={text}
333
- onChange={(e) => {
334
- setText(e.target.value)
335
- onActiveChange?.(e.target.value.length > 0)
336
- }}
337
- onKeyDown={handleKeyDown}
338
- placeholder={sending ? t('思考中...', 'Thinking...') : ''}
339
- disabled={sending}
340
- style={{ ...inputStyle, paddingLeft: 48, paddingRight: 80 }}
341
- autoFocus
342
- />
343
- <button
344
- onMouseDown={startRecording}
345
- onMouseUp={stopRecording}
346
- onMouseLeave={handleMouseLeave}
347
- style={{ ...inlineBtnRight, right: 44, color: recording ? 'rgba(255, 80, 80, 0.9)' : 'rgba(255, 255, 255, 0.45)' }}
348
- title={t('按住说话', 'Hold to speak')}
349
- >
350
- <Mic size={22} />
351
- </button>
352
- <button
353
- onClick={() => text.trim() ? send() : closeBar()}
354
- disabled={sending}
355
- style={{ ...inlineBtnRight, right: 12, color: text.trim() ? 'rgba(100, 160, 255, 0.9)' : 'rgba(255, 255, 255, 0.45)' }}
356
- title={text.trim() ? t('发送 (Enter)', 'Send (Enter)') : t('收起 (Esc)', 'Collapse (Esc)')}
357
- >
358
- {sending ? <Loader size={22} style={{ animation: 'spin 1s linear infinite' }} /> : text.trim() ? <Send size={22} /> : <ChevronDown size={22} />}
359
- </button>
360
- </div>
361
  </div>
362
- </>
363
  )
364
  }
365
 
366
- const fabStyle: React.CSSProperties = {
367
- width: 32,
368
- height: 32,
369
- border: 'none',
370
- borderRadius: 6,
371
- background: 'rgba(125, 125, 125, 0.28)',
372
- backdropFilter: 'blur(6px)',
373
- color: 'rgba(255, 255, 255, 0.8)',
374
- fontSize: 16,
375
- cursor: 'pointer',
376
- display: 'flex',
377
- alignItems: 'center',
378
- justifyContent: 'center',
379
- zIndex: 300,
380
- pointerEvents: 'auto',
381
- }
382
-
383
  const barStyle: React.CSSProperties = {
384
  position: 'absolute',
385
  bottom: 16,
@@ -394,7 +202,7 @@ const barStyle: React.CSSProperties = {
394
  const inputStyle: React.CSSProperties = {
395
  width: '100%',
396
  height: 50,
397
- boxSizing: 'border-box' as const,
398
  border: '1px solid rgba(255, 255, 255, 0.2)',
399
  borderRadius: 25,
400
  background: 'rgba(0, 0, 0, 0.4)',
@@ -405,70 +213,3 @@ const inputStyle: React.CSSProperties = {
405
  outline: 'none',
406
  fontFamily: '"Segoe UI", "Microsoft YaHei", sans-serif',
407
  }
408
-
409
- const inlineBtnLeft: React.CSSProperties = {
410
- position: 'absolute',
411
- left: 12,
412
- top: '50%',
413
- transform: 'translateY(-50%)',
414
- width: 32,
415
- height: 32,
416
- border: 'none',
417
- borderRadius: 4,
418
- background: 'transparent',
419
- cursor: 'pointer',
420
- display: 'flex',
421
- alignItems: 'center',
422
- justifyContent: 'center',
423
- padding: 0,
424
- zIndex: 2,
425
- }
426
-
427
- const inlineBtnRight: React.CSSProperties = {
428
- position: 'absolute',
429
- top: '50%',
430
- transform: 'translateY(-50%)',
431
- width: 32,
432
- height: 32,
433
- border: 'none',
434
- borderRadius: 4,
435
- background: 'transparent',
436
- cursor: 'pointer',
437
- display: 'flex',
438
- alignItems: 'center',
439
- justifyContent: 'center',
440
- padding: 0,
441
- zIndex: 1,
442
- }
443
-
444
- const popupMenuStyle: React.CSSProperties = {
445
- position: 'absolute',
446
- left: 4,
447
- bottom: '100%',
448
- marginBottom: 6,
449
- background: 'rgba(30, 30, 40, 0.95)',
450
- backdropFilter: 'blur(12px)',
451
- borderRadius: 10,
452
- border: '1px solid rgba(255, 255, 255, 0.15)',
453
- boxShadow: '0 4px 16px rgba(0, 0, 0, 0.4)',
454
- padding: 4,
455
- display: 'flex',
456
- flexDirection: 'column',
457
- gap: 2,
458
- zIndex: 1000,
459
- }
460
-
461
- const popupItemStyle: React.CSSProperties = {
462
- display: 'flex',
463
- alignItems: 'center',
464
- gap: 8,
465
- padding: '8px 14px',
466
- border: 'none',
467
- borderRadius: 8,
468
- background: 'transparent',
469
- color: 'rgba(255, 255, 255, 0.8)',
470
- fontSize: 13,
471
- cursor: 'pointer',
472
- whiteSpace: 'nowrap',
473
- fontFamily: '"Segoe UI", "Microsoft YaHei", sans-serif',
474
- }
 
1
  import { useState, useRef, useCallback, useEffect } from 'react'
2
+ import { Phone, PhoneOff, Mic } from 'lucide-react'
3
  import { FRIEND_API } from '../api'
4
  import { useServerStt, type SttProvider } from '../hooks/useServerStt'
5
 
6
+ export function ChatInput({ uiAlign = 'right', language = 'zh', sttProvider = 'browser' }: { uiAlign?: 'left' | 'right'; language?: 'zh' | 'en'; sttProvider?: SttProvider }) {
 
 
7
  const t = (zh: string, en: string) => language === 'en' ? en : zh
8
+ const [voiceCallActive, setVoiceCallActive] = useState(false)
9
  const [text, setText] = useState('')
 
10
  const [recording, setRecording] = useState(false)
 
 
 
 
11
  const voiceCallActiveRef = useRef(false)
 
12
  const serverStt = useServerStt()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
13
  const displayTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null)
14
 
15
+ // --- Voice Call ---
16
  const startVoiceCall = useCallback(async () => {
17
  setVoiceCallActive(true)
18
  voiceCallActiveRef.current = true
19
  setText('')
 
20
 
21
  serverStt.startStreaming(
22
  sttProvider,
23
+ (partialText, _isFinal) => {
24
+ if (partialText.trim()) {
25
+ setText(partialText)
 
 
 
 
 
26
  if (displayTimerRef.current) clearTimeout(displayTimerRef.current)
27
  displayTimerRef.current = setTimeout(() => {
28
  if (voiceCallActiveRef.current) {
29
  setText('')
 
30
  }
31
  }, 3000)
32
  }
33
  },
34
  (err) => {
35
+ console.error('Voice call error:', err)
36
  if (voiceCallActiveRef.current) {
37
+ setText(`\u8bed\u97f3\u542f\u52a8\u5931\u8d25: ${err}`)
38
  setTimeout(() => {
39
  setText('')
40
+ endVoiceCall()
41
  }, 3000)
42
  }
43
  },
44
  language === 'en' ? 'en' : 'zh',
45
  )
46
  setRecording(true)
47
+ }, [sttProvider, serverStt, language])
48
 
 
49
  const endVoiceCall = useCallback(async () => {
50
+ if (!voiceCallActiveRef.current) return
51
  voiceCallActiveRef.current = false
52
  setVoiceCallActive(false)
53
  setRecording(false)
 
 
54
  const finalText = await serverStt.stopStreaming()
 
 
 
55
  if (finalText.trim()) {
56
+ // Send text via chat endpoint
57
+ fetch(`${FRIEND_API}/chat`, {
58
+ method: 'POST',
59
+ headers: { 'Content-Type': 'application/json' },
60
+ body: JSON.stringify({ message: finalText }),
61
+ }).catch(() => {})
62
  }
 
 
 
63
  setText('')
64
+ }, [serverStt])
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
65
 
66
  // Global keyboard shortcuts
67
  useEffect(() => {
68
  const onGlobalKeyDown = (e: KeyboardEvent) => {
69
+ if (e.target instanceof HTMLInputElement || e.target instanceof HTMLTextAreaElement) return
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
70
 
71
  if (e.key === 'F2') {
72
  e.preventDefault()
 
79
  }
80
  window.addEventListener('keydown', onGlobalKeyDown)
81
  return () => window.removeEventListener('keydown', onGlobalKeyDown)
82
+ }, [voiceCallActive, startVoiceCall, endVoiceCall])
83
 
84
+ // Broadcast recording state for VRM "listening" response
85
+ useEffect(() => {
86
+ (window as any).__userRecording = recording
87
+ return () => { (window as any).__userRecording = false }
88
+ }, [recording])
89
 
90
+ // Cleanup display timer
91
+ useEffect(() => {
92
+ return () => {
93
+ if (displayTimerRef.current) clearTimeout(displayTimerRef.current)
94
+ }
95
+ }, [])
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
96
 
97
  if (voiceCallActive) {
98
  return (
99
  <div style={barStyle}>
100
  <div style={{ flex: 1, position: 'relative' }}>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
101
  <div
102
  style={{
103
  ...inputStyle,
 
115
  </div>
116
  <span
117
  style={{
118
+ position: 'absolute',
119
+ top: '50%',
120
+ transform: 'translateY(-50%)',
121
  right: 44,
122
+ width: 32,
123
+ height: 32,
124
+ display: 'flex',
125
+ alignItems: 'center',
126
+ justifyContent: 'center',
127
  color: recording ? 'rgba(255, 80, 80, 0.9)' : 'rgba(80, 200, 120, 0.9)',
128
  animation: recording ? 'claw-pulse 0.8s ease-in-out infinite' : 'none',
129
  transition: 'color 0.2s',
 
134
  </span>
135
  <button
136
  onClick={endVoiceCall}
137
+ style={{
138
+ position: 'absolute',
139
+ top: '50%',
140
+ transform: 'translateY(-50%)',
141
+ right: 12,
142
+ width: 32,
143
+ height: 32,
144
+ border: 'none',
145
+ borderRadius: 4,
146
+ background: 'transparent',
147
+ cursor: 'pointer',
148
+ display: 'flex',
149
+ alignItems: 'center',
150
+ justifyContent: 'center',
151
+ padding: 0,
152
+ color: 'rgba(255, 80, 80, 0.9)',
153
+ }}
154
  title={t('挂断 (F2)', 'Hang up (F2)')}
155
  >
156
  <PhoneOff size={22} />
 
161
  }
162
 
163
  return (
164
+ <div style={{ position: 'absolute', bottom: 8, ...(uiAlign === 'left' ? { left: 12 } : { right: 12 }), zIndex: 300, pointerEvents: 'auto', display: 'flex', flexDirection: 'column', gap: 4, alignItems: 'center' }}>
165
+ <button
166
+ onClick={startVoiceCall}
167
+ style={{
168
+ width: 32,
169
+ height: 32,
170
+ border: 'none',
171
+ borderRadius: 6,
172
+ background: 'rgba(125, 125, 125, 0.28)',
173
+ backdropFilter: 'blur(6px)',
174
+ color: 'rgba(255, 255, 255, 0.8)',
175
+ fontSize: 16,
176
+ cursor: 'pointer',
177
+ display: 'flex',
178
+ alignItems: 'center',
179
+ justifyContent: 'center',
180
+ zIndex: 300,
181
+ pointerEvents: 'auto',
182
+ }}
183
+ title={t('语音通话 (F2)', 'Voice Call (F2)')}
184
+ >
185
+ <Phone size={16} />
186
+ </button>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
187
  </div>
 
188
  )
189
  }
190
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
191
  const barStyle: React.CSSProperties = {
192
  position: 'absolute',
193
  bottom: 16,
 
202
  const inputStyle: React.CSSProperties = {
203
  width: '100%',
204
  height: 50,
205
+ boxSizing: 'border-box',
206
  border: '1px solid rgba(255, 255, 255, 0.2)',
207
  borderRadius: 25,
208
  background: 'rgba(0, 0, 0, 0.4)',
 
213
  outline: 'none',
214
  fontFamily: '"Segoe UI", "Microsoft YaHei", sans-serif',
215
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
src/components/friend/frontend/components/HistoryPanel.tsx DELETED
@@ -1,177 +0,0 @@
1
- import { useState, useEffect, useRef, useCallback } from 'react'
2
- import { X } from 'lucide-react'
3
- import { FRIEND_API } from '../api'
4
-
5
- interface HistoryMessage {
6
- role: string
7
- content: string
8
- timestamp?: number
9
- }
10
-
11
- interface HistoryPanelProps {
12
- visible: boolean
13
- onClose: () => void
14
- language?: 'zh' | 'en'
15
- }
16
-
17
- export function HistoryPanel({ visible, onClose, language = 'zh' }: HistoryPanelProps) {
18
- const t = (zh: string, en: string) => language === 'en' ? en : zh
19
- const [messages, setMessages] = useState<HistoryMessage[]>([])
20
- const [agentName, setAgentName] = useState('')
21
- const [loading, setLoading] = useState(false)
22
- const scrollRef = useRef<HTMLDivElement>(null)
23
-
24
- // Drag state
25
- const [panelPos, setPanelPos] = useState<{ x: number; y: number }>({ x: 0, y: 0 })
26
- const dragRef = useRef<{ dragging: boolean; startX: number; startY: number; origX: number; origY: number }>({
27
- dragging: false, startX: 0, startY: 0, origX: 0, origY: 0,
28
- })
29
-
30
- const onDragStart = useCallback((e: React.MouseEvent) => {
31
- dragRef.current = { dragging: true, startX: e.clientX, startY: e.clientY, origX: panelPos.x, origY: panelPos.y }
32
- const onMove = (ev: MouseEvent) => {
33
- if (!dragRef.current.dragging) return
34
- setPanelPos({
35
- x: dragRef.current.origX + ev.clientX - dragRef.current.startX,
36
- y: dragRef.current.origY + ev.clientY - dragRef.current.startY,
37
- })
38
- }
39
- const onUp = () => {
40
- dragRef.current.dragging = false
41
- window.removeEventListener('mousemove', onMove)
42
- window.removeEventListener('mouseup', onUp)
43
- }
44
- window.addEventListener('mousemove', onMove)
45
- window.addEventListener('mouseup', onUp)
46
- }, [panelPos])
47
-
48
- useEffect(() => {
49
- if (visible) setPanelPos({ x: 0, y: 0 })
50
- }, [visible])
51
-
52
- // Fetch history when panel opens
53
- useEffect(() => {
54
- if (!visible) return
55
- setLoading(true)
56
- fetch(`${FRIEND_API}/history`)
57
- .then((r) => r.json())
58
- .then((data) => {
59
- setMessages(data.messages || [])
60
- if (data.agentName) setAgentName(data.agentName)
61
- setTimeout(() => {
62
- if (scrollRef.current) scrollRef.current.scrollTop = scrollRef.current.scrollHeight
63
- }, 50)
64
- })
65
- .catch(() => setMessages([]))
66
- .finally(() => setLoading(false))
67
- }, [visible])
68
-
69
- if (!visible) return null
70
-
71
- return (
72
- <div style={overlayStyle} data-no-passthrough onClick={onClose}>
73
- <div style={{ ...panelStyle, transform: `translate(${panelPos.x}px, ${panelPos.y}px)` }} data-no-passthrough onClick={(e) => e.stopPropagation()}>
74
- <div style={headerStyle} onMouseDown={onDragStart}>
75
- <span style={{ fontSize: 16, fontWeight: 600, cursor: 'grab' }}>{t('对话历史', 'Chat History')}</span>
76
- <button onClick={onClose} style={closeBtnStyle}>
77
- <X size={16} />
78
- </button>
79
- </div>
80
-
81
- <div ref={scrollRef} style={messagesContainerStyle}>
82
- {loading && (
83
- <div style={{ textAlign: 'center', color: 'rgba(255,255,255,0.4)', padding: 20 }}>{t('加载中...', 'Loading...')}</div>
84
- )}
85
- {!loading && messages.length === 0 && (
86
- <div style={{ textAlign: 'center', color: 'rgba(255,255,255,0.4)', padding: 20 }}>{t('暂无对话记录', 'No chat history')}</div>
87
- )}
88
- {messages.map((msg, i) => (
89
- <div key={i} style={{ display: 'flex', flexDirection: 'column', alignItems: msg.role === 'user' ? 'flex-end' : 'flex-start' }}>
90
- <div style={{ fontSize: 11, color: 'rgba(255,255,255,0.35)', marginBottom: 2 }}>
91
- {msg.role === 'user' ? t('你', 'You') : (agentName || 'AI')}
92
- </div>
93
- <div style={{
94
- ...bubbleStyle,
95
- background: msg.role === 'user' ? 'rgba(100, 160, 255, 0.3)' : 'rgba(255, 255, 255, 0.08)',
96
- borderColor: msg.role === 'user' ? 'rgba(100, 160, 255, 0.4)' : 'rgba(255, 255, 255, 0.12)',
97
- }}>
98
- {msg.content}
99
- </div>
100
- </div>
101
- ))}
102
- </div>
103
- </div>
104
- </div>
105
- )
106
- }
107
-
108
- const overlayStyle: React.CSSProperties = {
109
- position: 'fixed',
110
- top: 0,
111
- left: 0,
112
- right: 0,
113
- bottom: 0,
114
- background: 'rgba(0, 0, 0, 0.5)',
115
- zIndex: 1000,
116
- display: 'flex',
117
- alignItems: 'center',
118
- justifyContent: 'center',
119
- pointerEvents: 'auto',
120
- }
121
-
122
- const panelStyle: React.CSSProperties = {
123
- width: 360,
124
- maxHeight: '80vh',
125
- background: 'rgba(30, 30, 40, 0.95)',
126
- backdropFilter: 'blur(12px)',
127
- borderRadius: 12,
128
- border: '1px solid rgba(255, 255, 255, 0.15)',
129
- boxShadow: '0 8px 32px rgba(0, 0, 0, 0.5)',
130
- padding: 16,
131
- color: '#fff',
132
- fontFamily: '"Segoe UI", "Microsoft YaHei", sans-serif',
133
- display: 'flex',
134
- flexDirection: 'column',
135
- }
136
-
137
- const headerStyle: React.CSSProperties = {
138
- display: 'flex',
139
- justifyContent: 'space-between',
140
- alignItems: 'center',
141
- marginBottom: 12,
142
- cursor: 'grab',
143
- userSelect: 'none',
144
- }
145
-
146
- const closeBtnStyle: React.CSSProperties = {
147
- width: 28,
148
- height: 28,
149
- border: 'none',
150
- borderRadius: 6,
151
- background: 'rgba(255, 255, 255, 0.1)',
152
- color: 'rgba(255, 255, 255, 0.7)',
153
- cursor: 'pointer',
154
- display: 'flex',
155
- alignItems: 'center',
156
- justifyContent: 'center',
157
- }
158
-
159
- const messagesContainerStyle: React.CSSProperties = {
160
- flex: 1,
161
- overflowY: 'auto',
162
- display: 'flex',
163
- flexDirection: 'column',
164
- gap: 12,
165
- maxHeight: 'calc(80vh - 60px)',
166
- }
167
-
168
- const bubbleStyle: React.CSSProperties = {
169
- maxWidth: '85%',
170
- padding: '8px 12px',
171
- borderRadius: 10,
172
- border: '1px solid',
173
- fontSize: 13,
174
- lineHeight: 1.5,
175
- wordBreak: 'break-word',
176
- whiteSpace: 'pre-wrap',
177
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
src/components/friend/frontend/components/MoodIndicator.tsx DELETED
@@ -1,372 +0,0 @@
1
- import { FRIEND_API } from '../api'
2
- import { useState, useEffect, useRef, useCallback } from 'react'
3
-
4
- interface MoodBubble {
5
- id: number
6
- delta: number
7
- }
8
-
9
- interface MoodIndicatorProps {
10
- uiAlign?: 'left' | 'right'
11
- }
12
-
13
- const BAR_WIDTH = 10
14
- const BAR_HEIGHT = 120
15
- const BORDER_GAP = 4 // gap between bar and border
16
- const BORDER_WIDTH = 2 // border stroke thickness
17
- const HEART_SIZE = 23
18
- const DPR = typeof window !== 'undefined' ? (window.devicePixelRatio || 1) : 1
19
-
20
- // Wave parameters (ported from Android HorizontalWaveProgressView)
21
- const WAVE_LENGTH = 8
22
- const WAVE_HEIGHT = 2.5
23
-
24
- // 5 tiers: 90+ pink, 70+ orange, 50+ green, 30+ blue, 0+ grey
25
- function moodRGB(percent: number): [number, number, number] {
26
- if (percent >= 90) return [255, 107, 157]
27
- if (percent >= 70) return [255, 165, 70]
28
- if (percent >= 50) return [72, 199, 142]
29
- if (percent >= 30) return [78, 168, 222]
30
- return [160, 168, 180]
31
- }
32
-
33
- /** Rounded-rect path helper */
34
- function roundedRectPath(ctx: CanvasRenderingContext2D, x: number, y: number, w: number, h: number, r: number) {
35
- ctx.beginPath()
36
- ctx.moveTo(x + r, y)
37
- ctx.lineTo(x + w - r, y)
38
- ctx.arc(x + w - r, y + r, r, -Math.PI / 2, 0)
39
- ctx.lineTo(x + w, y + h - r)
40
- ctx.arc(x + w - r, y + h - r, r, 0, Math.PI / 2)
41
- ctx.lineTo(x + r, y + h)
42
- ctx.arc(x + r, y + h - r, r, Math.PI / 2, Math.PI)
43
- ctx.lineTo(x, y + r)
44
- ctx.arc(x + r, y + r, r, Math.PI, Math.PI * 1.5)
45
- ctx.closePath()
46
- }
47
-
48
- /** Heart path centered at (cx, cy) with given size — classic round heart */
49
- function heartPath(ctx: CanvasRenderingContext2D, cx: number, cy: number, size: number) {
50
- const r = size * 0.45
51
- const topY = cy - r * 0.4
52
- ctx.beginPath()
53
- // Bottom tip (rounded)
54
- ctx.moveTo(cx, cy + r * 0.85)
55
- // Left curve
56
- ctx.bezierCurveTo(cx - r * 0.5, cy + r * 0.6, cx - r * 1.1, cy + r * 0.1, cx - r, topY)
57
- // Left bump (arc)
58
- ctx.arc(cx - r * 0.5, topY, r * 0.5, Math.PI, 0, false)
59
- // Right bump (arc)
60
- ctx.arc(cx + r * 0.5, topY, r * 0.5, Math.PI, 0, false)
61
- // Right curve
62
- ctx.bezierCurveTo(cx + r * 1.1, cy + r * 0.1, cx + r * 0.5, cy + r * 0.6, cx, cy + r * 0.85)
63
- ctx.closePath()
64
- }
65
-
66
- /**
67
- * Common wave drawing logic used by both the bar and the heart.
68
- * Draws dual bezier waves inside the current clip region.
69
- */
70
- function drawWaves(
71
- ctx: CanvasRenderingContext2D,
72
- regionWidth: number,
73
- regionHeight: number,
74
- waterY: number,
75
- moveDistance: number,
76
- cr: number, cg: number, cb: number,
77
- ) {
78
- const fullCycle = WAVE_LENGTH * 2
79
- const waveNumber = Math.ceil(regionWidth / fullCycle) + 2
80
-
81
- const drawWave = (scrollDir: number, alpha: number) => {
82
- ctx.beginPath()
83
- const offset = (moveDistance * scrollDir) % fullCycle
84
- let x = -fullCycle + offset
85
- ctx.moveTo(x, waterY)
86
-
87
- for (let i = 0; i < waveNumber * 2; i++) {
88
- ctx.quadraticCurveTo(x + WAVE_LENGTH / 2, waterY - WAVE_HEIGHT, x + WAVE_LENGTH, waterY)
89
- x += WAVE_LENGTH
90
- ctx.quadraticCurveTo(x + WAVE_LENGTH / 2, waterY + WAVE_HEIGHT, x + WAVE_LENGTH, waterY)
91
- x += WAVE_LENGTH
92
- }
93
-
94
- ctx.lineTo(x, regionHeight)
95
- ctx.lineTo(-fullCycle, regionHeight)
96
- ctx.closePath()
97
- ctx.fillStyle = `rgba(${cr},${cg},${cb},${alpha})`
98
- ctx.fill()
99
- }
100
-
101
- drawWave(-1, 0.55)
102
- drawWave(1, 1.0)
103
- }
104
-
105
- /**
106
- * Draw the vertical liquid-filled bar with border that has a gap.
107
- */
108
- function drawLiquidBar(
109
- ctx: CanvasRenderingContext2D,
110
- w: number,
111
- h: number,
112
- percent: number,
113
- moveDistance: number,
114
- ) {
115
- const radius = w / 2
116
- const totalW = w + (BORDER_GAP + BORDER_WIDTH) * 2
117
- const totalH = h + (BORDER_GAP + BORDER_WIDTH) * 2
118
- const offsetX = BORDER_GAP + BORDER_WIDTH
119
- const offsetY = BORDER_GAP + BORDER_WIDTH
120
-
121
- ctx.clearRect(0, 0, totalW, totalH)
122
-
123
- const waterY = h * (1 - percent / 100)
124
- const [cr, cg, cb] = moodRGB(percent)
125
-
126
- // ── Bar fill ──
127
- ctx.save()
128
- roundedRectPath(ctx, offsetX, offsetY, w, h, radius)
129
- ctx.clip()
130
-
131
- ctx.clearRect(offsetX, offsetY, w, h)
132
-
133
- ctx.save()
134
- ctx.translate(offsetX, offsetY)
135
- drawWaves(ctx, w, h, waterY, moveDistance, cr, cg, cb)
136
- ctx.restore()
137
-
138
- ctx.restore()
139
-
140
- // ── Border with gap ──
141
- const bx = offsetX - BORDER_GAP - BORDER_WIDTH / 2
142
- const by = offsetY - BORDER_GAP - BORDER_WIDTH / 2
143
- const bw = w + (BORDER_GAP + BORDER_WIDTH / 2) * 2
144
- const bh = h + (BORDER_GAP + BORDER_WIDTH / 2) * 2
145
- const br = radius + BORDER_GAP + BORDER_WIDTH / 2
146
- roundedRectPath(ctx, bx, by, bw, bh, br)
147
- ctx.strokeStyle = 'rgba(80, 80, 85, 0.75)'
148
- ctx.lineWidth = BORDER_WIDTH
149
- ctx.stroke()
150
- }
151
-
152
- /**
153
- * Draw liquid-filled heart icon with border and gap (matching bar style).
154
- */
155
- function drawLiquidHeart(
156
- ctx: CanvasRenderingContext2D,
157
- size: number,
158
- percent: number,
159
- moveDistance: number,
160
- ) {
161
- ctx.clearRect(0, 0, size, size)
162
-
163
- const cx = size / 2
164
- const cy = size / 2
165
- const waterY = size * (1 - percent / 100)
166
- const [cr, cg, cb] = moodRGB(percent)
167
-
168
- // ── Heart fill ──
169
- ctx.save()
170
- heartPath(ctx, cx, cy, size)
171
- ctx.clip()
172
-
173
- ctx.fillStyle = 'rgba(60, 60, 65, 0.9)'
174
- ctx.fillRect(0, 0, size, size)
175
-
176
- drawWaves(ctx, size, size, waterY, moveDistance, cr, cg, cb)
177
-
178
- ctx.restore()
179
- }
180
-
181
- let bubbleIdCounter = 0
182
-
183
- // Canvas total dimensions (bar + border gap)
184
- const BAR_CANVAS_W = BAR_WIDTH + (BORDER_GAP + BORDER_WIDTH) * 2
185
- const BAR_CANVAS_H = BAR_HEIGHT + (BORDER_GAP + BORDER_WIDTH) * 2
186
-
187
- export function MoodIndicator({ uiAlign = 'right' }: MoodIndicatorProps) {
188
- const [mood, setMood] = useState(60)
189
- const [bubbles, setBubbles] = useState<MoodBubble[]>([])
190
- const [visible, setVisible] = useState(false)
191
- const [dragging, setDragging] = useState(false)
192
- const defaultX = (align: string) => align === 'left' ? window.innerWidth - 20 - BAR_CANVAS_W : 20
193
- const [pos, setPos] = useState<{ x: number; y: number }>({ x: defaultX(uiAlign), y: 20 })
194
- const [userDragged, setUserDragged] = useState(false)
195
- const hideTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null)
196
- const containerRef = useRef<HTMLDivElement>(null)
197
- const barCanvasRef = useRef<HTMLCanvasElement>(null)
198
- const heartCanvasRef = useRef<HTMLCanvasElement>(null)
199
- const animRef = useRef<number>(0)
200
- const displayPercentRef = useRef(60)
201
- const moveDistRef = useRef(0)
202
-
203
- useEffect(() => {
204
- if (!userDragged) setPos(p => ({ ...p, x: defaultX(uiAlign) }))
205
- }, [uiAlign, userDragged])
206
-
207
- useEffect(() => {
208
- fetch(`${FRIEND_API}/settings`)
209
- .then((r) => r.json())
210
- .then((s) => { if (s.moodIndex !== undefined) { setMood(s.moodIndex); displayPercentRef.current = s.moodIndex } })
211
- .catch(() => {})
212
- }, [])
213
-
214
- const showBriefly = useCallback(() => {
215
- setVisible(true)
216
- if (hideTimerRef.current) clearTimeout(hideTimerRef.current)
217
- hideTimerRef.current = setTimeout(() => setVisible(false), 5000)
218
- }, [])
219
-
220
- useEffect(() => {
221
- const es = new EventSource(`${FRIEND_API}/events`)
222
- es.onmessage = (e) => {
223
- try {
224
- const data = JSON.parse(e.data)
225
- if (data.moodIndex !== undefined && data.moodDelta !== undefined) {
226
- setMood(data.moodIndex)
227
- showBriefly()
228
- const id = ++bubbleIdCounter
229
- setBubbles((prev) => [...prev, { id, delta: data.moodDelta }])
230
- setTimeout(() => setBubbles((prev) => prev.filter((b) => b.id !== id)), 2000)
231
- }
232
- } catch { /* ignore */ }
233
- }
234
- return () => es.close()
235
- }, [showBriefly])
236
-
237
- // Canvas animation loop — draws both heart and bar
238
- useEffect(() => {
239
- const barCanvas = barCanvasRef.current
240
- const heartCanvas = heartCanvasRef.current
241
- if (!barCanvas || !heartCanvas) return
242
- const barCtx = barCanvas.getContext('2d')
243
- const heartCtx = heartCanvas.getContext('2d')
244
- if (!barCtx || !heartCtx) return
245
-
246
- barCanvas.width = BAR_CANVAS_W * DPR
247
- barCanvas.height = BAR_CANVAS_H * DPR
248
- barCtx.scale(DPR, DPR)
249
-
250
- heartCanvas.width = HEART_SIZE * DPR
251
- heartCanvas.height = HEART_SIZE * DPR
252
- heartCtx.scale(DPR, DPR)
253
-
254
- let running = true
255
- const animate = () => {
256
- if (!running) return
257
- const target = Math.max(2, mood)
258
- displayPercentRef.current += (target - displayPercentRef.current) * 0.08
259
- moveDistRef.current += 0.12
260
-
261
- drawLiquidBar(barCtx, BAR_WIDTH, BAR_HEIGHT, displayPercentRef.current, moveDistRef.current)
262
- drawLiquidHeart(heartCtx, HEART_SIZE, displayPercentRef.current, moveDistRef.current)
263
- animRef.current = requestAnimationFrame(animate)
264
- }
265
- animate()
266
- return () => { running = false; cancelAnimationFrame(animRef.current) }
267
- }, [mood])
268
-
269
- const handlePointerDown = useCallback((e: React.PointerEvent) => {
270
- e.preventDefault()
271
- e.stopPropagation()
272
- setDragging(true)
273
- setUserDragged(true)
274
- setVisible(true)
275
- if (hideTimerRef.current) { clearTimeout(hideTimerRef.current); hideTimerRef.current = null }
276
-
277
- const el = containerRef.current
278
- if (!el) return
279
- const rect = el.getBoundingClientRect()
280
- const offsetX = e.clientX - rect.left
281
- const offsetY = e.clientY - rect.top
282
-
283
- const onMove = (ev: PointerEvent) => {
284
- setPos({ x: ev.clientX - offsetX, y: ev.clientY - offsetY })
285
- }
286
- const onUp = () => {
287
- setDragging(false)
288
- window.removeEventListener('pointermove', onMove)
289
- window.removeEventListener('pointerup', onUp)
290
- hideTimerRef.current = setTimeout(() => setVisible(false), 3000)
291
- }
292
- window.addEventListener('pointermove', onMove)
293
- window.addEventListener('pointerup', onUp)
294
- }, [])
295
-
296
- const handleMouseEnter = useCallback(() => {
297
- setVisible(true)
298
- if (hideTimerRef.current) { clearTimeout(hideTimerRef.current); hideTimerRef.current = null }
299
- }, [])
300
-
301
- const handleMouseLeave = useCallback(() => {
302
- if (!dragging) {
303
- hideTimerRef.current = setTimeout(() => setVisible(false), 2000)
304
- }
305
- }, [dragging])
306
-
307
- return (
308
- <div
309
- ref={containerRef}
310
- style={{
311
- position: 'absolute',
312
- top: pos.y,
313
- left: pos.x,
314
- zIndex: 250,
315
- pointerEvents: 'auto',
316
- display: 'flex',
317
- flexDirection: 'column',
318
- alignItems: 'center',
319
- transition: 'opacity 0.5s ease',
320
- opacity: visible || bubbles.length > 0 ? 1 : 0.5,
321
- cursor: dragging ? 'grabbing' : 'grab',
322
- userSelect: 'none',
323
- }}
324
- onMouseEnter={handleMouseEnter}
325
- onMouseLeave={handleMouseLeave}
326
- onPointerDown={handlePointerDown}
327
- data-no-passthrough
328
- >
329
- {/* Liquid-filled heart */}
330
- <canvas
331
- ref={heartCanvasRef}
332
- width={HEART_SIZE * DPR}
333
- height={HEART_SIZE * DPR}
334
- style={{ width: HEART_SIZE, height: HEART_SIZE, display: 'block', marginBottom: 4 }}
335
- />
336
-
337
- {/* Vertical liquid bar with border */}
338
- <div style={{ position: 'relative', width: BAR_CANVAS_W, height: BAR_CANVAS_H }}>
339
- <canvas
340
- ref={barCanvasRef}
341
- width={BAR_CANVAS_W * DPR}
342
- height={BAR_CANVAS_H * DPR}
343
- style={{ width: BAR_CANVAS_W, height: BAR_CANVAS_H, display: 'block' }}
344
- />
345
- {/* Floating bubbles */}
346
- {bubbles.map((b) => (
347
- <span
348
- key={b.id}
349
- style={{
350
- position: 'absolute',
351
- left: '50%',
352
- top: -20,
353
- transform: 'translateX(-50%)',
354
- fontSize: 14,
355
- fontWeight: 800,
356
- color: b.delta > 0 ? '#FF6B9D' : '#7EB0D5',
357
- whiteSpace: 'nowrap',
358
- pointerEvents: 'none',
359
- animation: 'mood-bubble-float 2.5s ease-out forwards',
360
- fontFamily: '"Segoe UI", "Microsoft YaHei", sans-serif',
361
- textShadow: b.delta > 0
362
- ? '0 0 6px rgba(255,107,157,0.6), 0 1px 3px rgba(0,0,0,0.4)'
363
- : '0 0 6px rgba(126,176,213,0.6), 0 1px 3px rgba(0,0,0,0.4)',
364
- }}
365
- >
366
- {b.delta > 0 ? `❤️+${b.delta}` : `🩶${b.delta}`}
367
- </span>
368
- ))}
369
- </div>
370
- </div>
371
- )
372
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
src/components/friend/frontend/components/SettingsPanel.tsx CHANGED
@@ -1,16 +1,6 @@
1
  import { useState, useEffect, useRef, useCallback } from 'react'
2
- import { X, Play, Loader, Sparkles, Trash2, Upload, Music } from 'lucide-react'
3
  import { FRIEND_API } from '../api'
4
- import { dancePresets, type DancePreset } from '../motion-controller'
5
- import { pickFile } from '../web-file-picker'
6
-
7
- interface DanceItem {
8
- id: string
9
- label: string
10
- vmdUrl: string
11
- bgmUrl?: string
12
- builtin?: boolean
13
- }
14
 
15
  type SttProvider = 'browser' | 'groq' | 'anthropic' | 'local' | 'doubao'
16
 
@@ -31,23 +21,13 @@ interface SettingsPanelProps {
31
  onVolumeChange: (v: number) => void
32
  uiAlign: 'left' | 'right'
33
  onUiAlignChange: (v: 'left' | 'right') => void
34
- hideMood: boolean
35
- onHideMoodChange: (v: boolean) => void
36
- screenObserve: boolean
37
- onScreenObserveChange: (v: boolean) => void
38
- screenObserveInterval: number
39
- onScreenObserveIntervalChange: (v: number) => void
40
- /** Return a data URL screenshot of the current VRM canvas */
41
- captureVrmScreenshot?: () => string | null
42
  language: 'zh' | 'en'
43
  onLanguageChange: (v: 'zh' | 'en') => void
44
- currentDance: string
45
- onDanceChange: (id: string, preset?: DancePreset) => void
46
  sttProvider?: SttProvider
47
  onSttProviderChange?: (v: SttProvider) => void
48
  }
49
 
50
- type Tab = 'general' | 'voice' | 'model' | 'persona' | 'dance'
51
 
52
  const BUILTIN_MODELS = ['/friend/model1.vrm', '/friend/model2.vrm', '/friend/model3.vrm', '/friend/model4.vrm', '/friend/model5.vrm']
53
 
@@ -103,32 +83,18 @@ export function SettingsPanel({
103
  tracking, onTrackingChange,
104
  volume, onVolumeChange,
105
  uiAlign, onUiAlignChange,
106
- hideMood, onHideMoodChange,
107
- screenObserve, onScreenObserveChange,
108
- screenObserveInterval, onScreenObserveIntervalChange,
109
- captureVrmScreenshot,
110
  language, onLanguageChange,
111
- currentDance, onDanceChange,
112
  sttProvider = 'browser', onSttProviderChange,
113
  }: SettingsPanelProps) {
114
  const t = (zh: string, en: string) => language === 'en' ? en : zh
115
 
116
  const [tab, setTab] = useState<Tab>('general')
117
- const [models, setModels] = useState<string[]>([])
118
- const [soulContent, setSoulContent] = useState('')
119
- const [identityContent, setIdentityContent] = useState('')
120
- const [personaDirty, setPersonaDirty] = useState(false)
121
- const [personaSaving, setPersonaSaving] = useState(false)
122
- const [generating, setGenerating] = useState(false)
123
  const [currentVoice, setCurrentVoice] = useState('')
124
  const [currentProvider, setCurrentProvider] = useState<string>('edge')
125
  const [qwenKey, setQwenKey] = useState('')
126
  const [qwenModel, setQwenModel] = useState('qwen3-tts-flash')
127
  const [previewingId, setPreviewingId] = useState<string | null>(null)
128
  const audioRef = useRef<HTMLAudioElement | null>(null)
129
- const [customDances, setCustomDances] = useState<DanceItem[]>([])
130
- const [importingDance, setImportingDance] = useState(false)
131
-
132
 
133
  // Drag state
134
  const [panelPos, setPanelPos] = useState<{ x: number; y: number }>({ x: 0, y: 0 })
@@ -154,18 +120,12 @@ export function SettingsPanel({
154
  window.addEventListener('mouseup', onUp)
155
  }, [panelPos])
156
 
157
- // Reset position when panel opens
158
  useEffect(() => {
159
  if (visible) setPanelPos({ x: 0, y: 0 })
160
  }, [visible])
161
 
162
  useEffect(() => {
163
  if (!visible) return
164
- fetch(`${FRIEND_API}/model/list`)
165
- .then((r) => r.json())
166
- .then((data) => { if (data.models) setModels(data.models) })
167
- .catch(() => setModels([]))
168
- // Fetch current voice + provider
169
  fetch(`${FRIEND_API}/voice`)
170
  .then((r) => r.json())
171
  .then((data) => {
@@ -177,72 +137,6 @@ export function SettingsPanel({
177
  .catch(() => {})
178
  }, [visible])
179
 
180
- // Fetch persona files when model tab / persona sub-tab is active
181
- useEffect(() => {
182
- if (!visible || tab !== 'persona') return
183
- fetch(`${FRIEND_API}/persona`)
184
- .then((r) => r.json())
185
- .then((data) => {
186
- setSoulContent(data.soul || '')
187
- setIdentityContent(data.identity || '')
188
- setPersonaDirty(false)
189
- })
190
- .catch(() => {})
191
- }, [visible, tab])
192
-
193
- const fetchCustomDances = useCallback(() => {
194
- fetch(`${FRIEND_API}/dance/list`)
195
- .then((r) => r.json())
196
- .then((data) => { if (data.dances) setCustomDances(data.dances) })
197
- .catch(() => setCustomDances([]))
198
- }, [])
199
-
200
- useEffect(() => {
201
- if (!visible || tab !== 'dance') return
202
- fetchCustomDances()
203
- }, [visible, tab, fetchCustomDances])
204
-
205
- const savePersona = useCallback(() => {
206
- setPersonaSaving(true)
207
- fetch(`${FRIEND_API}/persona`, {
208
- method: 'POST',
209
- headers: { 'Content-Type': 'application/json' },
210
- body: JSON.stringify({ soul: soulContent, identity: identityContent }),
211
- })
212
- .then(() => { setPersonaDirty(false) })
213
- .catch(() => {})
214
- .finally(() => setPersonaSaving(false))
215
- }, [soulContent, identityContent])
216
-
217
- const generatePersona = useCallback(async () => {
218
- if (!captureVrmScreenshot) return
219
- const dataUrl = captureVrmScreenshot()
220
- if (!dataUrl) return
221
- setGenerating(true)
222
- try {
223
- // Save screenshot to server first
224
- const saveRes = await fetch(`${FRIEND_API}/persona/screenshot`, {
225
- method: 'POST',
226
- headers: { 'Content-Type': 'application/json' },
227
- body: JSON.stringify({ image: dataUrl }),
228
- })
229
- const saveData = await saveRes.json()
230
- if (!saveData.ok) throw new Error(saveData.error || 'save screenshot failed')
231
- // Generate persona from saved screenshot
232
- const genRes = await fetch(`${FRIEND_API}/persona/generate`, {
233
- method: 'POST',
234
- headers: { 'Content-Type': 'application/json' },
235
- })
236
- const genData = await genRes.json()
237
- if (genData.soul || genData.identity) {
238
- if (genData.soul) setSoulContent(genData.soul)
239
- if (genData.identity) setIdentityContent(genData.identity)
240
- setPersonaDirty(true)
241
- }
242
- } catch { /* ignore */ }
243
- setGenerating(false)
244
- }, [captureVrmScreenshot])
245
-
246
  const postVoiceSettings = (body: Record<string, string | undefined>) => {
247
  return fetch(`${FRIEND_API}/voice`, {
248
  method: 'POST',
@@ -286,7 +180,6 @@ export function SettingsPanel({
286
  }, [])
287
 
288
  const preview = (voiceId: string) => {
289
- // Stop any current preview first
290
  stopPreview()
291
  setPreviewingId(voiceId)
292
  fetch(`${FRIEND_API}/preview`, {
@@ -326,13 +219,13 @@ export function SettingsPanel({
326
 
327
  {/* Tabs */}
328
  <div style={tabBarStyle}>
329
- {(['general', 'voice', 'model', 'persona', 'dance'] as const).map((tb) => (
330
  <button
331
  key={tb}
332
  onClick={() => setTab(tb)}
333
  style={{ ...tabStyle, ...(tab === tb ? activeTabStyle : {}) }}
334
  >
335
- {{ general: t('常规', 'General'), voice: t('语音', 'Voice'), model: t('形象', 'Model'), persona: t('人设', 'Persona'), dance: t('舞蹈', 'Dance') }[tb]}
336
  </button>
337
  ))}
338
  </div>
@@ -341,99 +234,13 @@ export function SettingsPanel({
341
  <div style={contentStyle}>
342
  {tab === 'general' && (
343
  <div style={{ display: 'flex', flexDirection: 'column', gap: 16 }}>
344
- <div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center' }}>
345
- <span style={{ fontSize: 14 }}>{t('语言', 'Language')}</span>
346
- <div style={{ display: 'flex', gap: 4 }}>
347
- {(['zh', 'en'] as const).map((l) => (
348
- <button
349
- key={l}
350
- onClick={() => onLanguageChange(l)}
351
- style={{
352
- ...smallBtnStyle,
353
- background: language === l ? 'rgba(100, 160, 255, 0.4)' : 'rgba(255, 255, 255, 0.08)',
354
- borderColor: language === l ? 'rgba(100, 160, 255, 0.6)' : 'rgba(255, 255, 255, 0.15)',
355
- }}
356
- >
357
- {l === 'zh' ? '中文' : 'English'}
358
- </button>
359
- ))}
360
- </div>
361
- </div>
362
  <ToggleRow label={t('显示字幕', 'Subtitles')} value={showText} onChange={onShowTextChange} />
363
  <ToggleRow label={t('语音播报', 'TTS')} value={ttsEnabled} onChange={onTtsEnabledChange} />
364
- <div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center' }}>
365
- <span style={{ fontSize: 14 }}>{t('音量', 'Volume')}</span>
366
- <div style={{ display: 'flex', alignItems: 'center', gap: 8 }}>
367
- <input
368
- type="range"
369
- min={0}
370
- max={100}
371
- value={Math.round(volume * 100)}
372
- onChange={(e) => onVolumeChange(Number(e.target.value) / 100)}
373
- style={{ width: 100, accentColor: 'rgba(100, 160, 255, 0.8)' }}
374
- />
375
- <span style={{ fontSize: 12, color: 'rgba(255,255,255,0.5)', width: 28, textAlign: 'right' }}>{Math.round(volume * 100)}</span>
376
- </div>
377
- </div>
378
- <div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center' }}>
379
- <span style={{ fontSize: 14 }}>{t('视线跟随', 'Eye Tracking')}</span>
380
- <div style={{ display: 'flex', gap: 4 }}>
381
- {(['mouse', 'camera'] as const).map((m) => (
382
- <button
383
- key={m}
384
- onClick={() => onTrackingChange(m)}
385
- style={{
386
- ...smallBtnStyle,
387
- background: tracking === m ? 'rgba(100, 160, 255, 0.4)' : 'rgba(255, 255, 255, 0.08)',
388
- borderColor: tracking === m ? 'rgba(100, 160, 255, 0.6)' : 'rgba(255, 255, 255, 0.15)',
389
- }}
390
- >
391
- {m === 'mouse' ? t('鼠标', 'Mouse') : t('镜头', 'Camera')}
392
- </button>
393
- ))}
394
- </div>
395
- </div>
396
- <div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center' }}>
397
- <span style={{ fontSize: 14 }}>{t('UI位置', 'UI Position')}</span>
398
- <div style={{ display: 'flex', gap: 4 }}>
399
- {(['left', 'right'] as const).map((a) => (
400
- <button
401
- key={a}
402
- onClick={() => onUiAlignChange(a)}
403
- style={{
404
- ...smallBtnStyle,
405
- background: uiAlign === a ? 'rgba(100, 160, 255, 0.4)' : 'rgba(255, 255, 255, 0.08)',
406
- borderColor: uiAlign === a ? 'rgba(100, 160, 255, 0.6)' : 'rgba(255, 255, 255, 0.15)',
407
- }}
408
- >
409
- {a === 'left' ? t('靠左', 'Left') : t('靠右', 'Right')}
410
- </button>
411
- ))}
412
- </div>
413
- </div>
414
  <ToggleRow label={t('隐藏UI', 'Hide UI')} value={hideUI} onChange={onHideUIChange} />
415
- <ToggleRow label={t('隐藏心情条', 'Hide Mood Bar')} value={hideMood} onChange={onHideMoodChange} />
416
- <ToggleRow label={t('屏幕观察', 'Screen Observe')} value={screenObserve} onChange={onScreenObserveChange} />
417
- {screenObserve && (
418
- <>
419
- <div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center', marginTop: -8 }}>
420
- <span style={{ fontSize: 14 }}>{t('观察间隔', 'Interval')}</span>
421
- <div style={{ display: 'flex', alignItems: 'center', gap: 8 }}>
422
- <input
423
- type="range"
424
- min={15}
425
- max={300}
426
- step={15}
427
- value={screenObserveInterval}
428
- onChange={(e) => onScreenObserveIntervalChange(Number(e.target.value))}
429
- style={{ width: 100, accentColor: 'rgba(100, 160, 255, 0.8)' }}
430
- />
431
- <span style={{ fontSize: 12, color: 'rgba(255,255,255,0.5)', width: 36, textAlign: 'right' }}>{screenObserveInterval}s</span>
432
- </div>
433
- </div>
434
-
435
- </>
436
- )}
437
  </div>
438
  )}
439
 
@@ -513,11 +320,6 @@ export function SettingsPanel({
513
  </button>
514
  ))}
515
  </div>
516
- <div style={{ fontSize: 11, color: 'rgba(255,255,255,0.35)', marginTop: 4 }}>
517
- {sttProvider === 'browser'
518
- ? t('使用浏览器内置语音识别(Web Speech API)', 'Use browser built-in speech recognition (Web Speech API)')
519
- : t('使用服务端语音识别,需要登录对应服务', 'Use server-side speech recognition, requires corresponding service login')}
520
- </div>
521
  </div>
522
 
523
  <div style={{ marginTop: 8 }}>
@@ -555,7 +357,6 @@ export function SettingsPanel({
555
  cursor: previewingId !== null ? 'default' : 'pointer',
556
  flexShrink: 0,
557
  opacity: previewingId !== null && previewingId !== v.id ? 0.3 : 0.7,
558
- transition: 'background 0.15s',
559
  }}
560
  title={t('试听', 'Preview')}
561
  >
@@ -581,268 +382,97 @@ export function SettingsPanel({
581
  <option key={m} value={m}>{m.replace(/^\//, '')}</option>
582
  ))}
583
  </select>
584
-
585
- {models.length > 0 && (
586
- <div style={{ marginTop: 12 }}>
587
- <div style={labelStyle}>{t('自定义VRM模型', 'Custom VRM Models')}</div>
588
- <select
589
- value={!BUILTIN_MODELS.includes(currentModel) ? currentModel : ''}
590
- onChange={(e) => { onModelChange(e.target.value); saveModelPath(e.target.value) }}
591
- style={selectStyle}
592
- >
593
- {BUILTIN_MODELS.includes(currentModel) && <option value="" disabled>{t('未选择', 'Not selected')}</option>}
594
- {models.map((m) => (
595
- <option key={m} value={m}>{decodeURIComponent(m.split('/').pop() || m)}</option>
596
- ))}
597
- </select>
598
- </div>
599
- )}
600
-
601
- <div style={{ marginTop: 12 }}>
602
- <div style={labelStyle}>{t('导入自定义模型', 'Import Custom Model')}</div>
603
- <button
604
- onClick={async () => {
605
- const file = await pickFile('.vrm,.glb')
606
- if (!file) return
607
- try {
608
- const formData = new FormData()
609
- formData.append('file', file)
610
- const res = await fetch(`${FRIEND_API}/model/import`, {
611
- method: 'POST',
612
- body: formData,
613
- })
614
- const data = await res.json()
615
- if (data.url) {
616
- fetch(`${FRIEND_API}/model/list`)
617
- .then((r) => r.json())
618
- .then((d) => { if (d.models) setModels(d.models) })
619
- .catch(() => {})
620
- onModelChange(data.url)
621
- saveModelPath(data.url)
622
- }
623
- } catch (err) {
624
- console.warn('Import model failed:', err)
625
- }
626
- }}
627
- style={{ ...applyBtnStyle, width: '100%' }}
628
- >
629
- {t('浏览本地文件…', 'Browse local files…')}
630
- </button>
631
- <div style={{ fontSize: 11, color: 'rgba(255,255,255,0.35)', marginTop: 4 }}>
632
- {t('选择本地 .vrm 文件,将保存到工作区 models 目录', 'Select a .vrm file to save to workspace models directory')}
633
- </div>
634
- </div>
635
  </div>
636
  )}
 
 
 
 
 
637
 
638
- {tab === 'persona' && (
639
- <div style={sectionStyle}>
640
- <div style={labelStyle}>IDENTITY.md</div>
641
- <textarea
642
- value={identityContent}
643
- onChange={(e) => { setIdentityContent(e.target.value); setPersonaDirty(true) }}
644
- placeholder={t('角色身份信息(名字、种族、性格、emoji 等)…', 'Character identity (name, race, personality, emoji, etc.)…')}
645
- style={textareaStyle}
646
- rows={4}
647
- />
648
-
649
- <div style={{ ...labelStyle, marginTop: 4 }}>SOUL.md</div>
650
- <textarea
651
- value={soulContent}
652
- onChange={(e) => { setSoulContent(e.target.value); setPersonaDirty(true) }}
653
- placeholder={t('角色灵魂设定(说话风格、行为准则、背景故事等)…', 'Character soul (speaking style, behavior, backstory, etc.)…')}
654
- style={textareaStyle}
655
- rows={6}
656
- />
657
-
658
- <div style={{ display: 'flex', gap: 4, marginTop: 8 }}>
659
- <button
660
- onClick={generatePersona}
661
- disabled={generating || !captureVrmScreenshot}
662
- style={{
663
- ...applyBtnStyle,
664
- flex: 1,
665
- display: 'flex',
666
- alignItems: 'center',
667
- justifyContent: 'center',
668
- gap: 4,
669
- background: generating ? 'rgba(100, 160, 255, 0.3)' : 'rgba(160, 120, 255, 0.5)',
670
- }}
671
- title={t('根据当前模型截图自动生成人设', 'Auto-generate persona from model screenshot')}
672
- >
673
- {generating
674
- ? <Loader size={14} style={{ animation: 'spin 1s linear infinite' }} />
675
- : <Sparkles size={14} />}
676
- {generating ? t('生成中…', 'Generating…') : t('一键生成人设', 'Auto Generate')}
677
- </button>
678
- <button
679
- onClick={savePersona}
680
- disabled={!personaDirty || personaSaving}
681
- style={{
682
- ...applyBtnStyle,
683
- opacity: personaDirty ? 1 : 0.4,
684
- }}
685
- >
686
- {personaSaving ? t('保存中…', 'Saving…') : t('保存', 'Save')}
687
- </button>
688
- </div>
689
- <div style={{ fontSize: 11, color: 'rgba(255,255,255,0.3)', marginTop: 4 }}>
690
- {t('关联工作区根目录的 IDENTITY.md 和 SOUL.md', 'Linked to IDENTITY.md and SOUL.md in workspace root')}
691
- </div>
692
- </div>
693
- )}
694
-
695
- {tab === 'dance' && (
696
- <div style={sectionStyle}>
697
- <div style={labelStyle}>{t('内置舞蹈', 'Built-in Dances')}</div>
698
- <div style={{ display: 'flex', flexDirection: 'column', gap: 4 }}>
699
- {Object.entries(dancePresets).map(([id, preset]) => (
700
- <div
701
- key={id}
702
- onClick={() => onDanceChange(id)}
703
- style={{
704
- ...modelBtnStyle,
705
- display: 'flex',
706
- alignItems: 'center',
707
- justifyContent: 'space-between',
708
- padding: '8px 12px',
709
- fontSize: 13,
710
- background: currentDance === id ? 'rgba(100, 160, 255, 0.4)' : 'rgba(255, 255, 255, 0.08)',
711
- borderColor: currentDance === id ? 'rgba(100, 160, 255, 0.6)' : 'rgba(255, 255, 255, 0.15)',
712
- }}
713
- >
714
- <div style={{ display: 'flex', alignItems: 'center', gap: 8 }}>
715
- <Music size={14} style={{ opacity: 0.6 }} />
716
- <span>{preset.label}</span>
717
- </div>
718
- {preset.bgm && (
719
- <span style={{ fontSize: 11, color: 'rgba(255,255,255,0.35)' }}>{t('含BGM', 'w/ BGM')}</span>
720
- )}
721
- </div>
722
- ))}
723
- </div>
724
-
725
- {customDances.length > 0 && (
726
- <div style={{ marginTop: 12 }}>
727
- <div style={labelStyle}>{t('自定义舞蹈', 'Custom Dances')}</div>
728
- <div style={{ display: 'flex', flexDirection: 'column', gap: 4 }}>
729
- {customDances.map((dance) => (
730
- <div
731
- key={dance.id}
732
- onClick={() => onDanceChange(`custom:${dance.id}`, {
733
- label: dance.label,
734
- type: 'vmd',
735
- url: dance.vmdUrl,
736
- bgm: dance.bgmUrl,
737
- })}
738
- style={{
739
- ...modelBtnStyle,
740
- display: 'flex',
741
- alignItems: 'center',
742
- justifyContent: 'space-between',
743
- padding: '8px 10px',
744
- fontSize: 13,
745
- background: currentDance === `custom:${dance.id}` ? 'rgba(100, 160, 255, 0.4)' : 'rgba(255, 255, 255, 0.08)',
746
- borderColor: currentDance === `custom:${dance.id}` ? 'rgba(100, 160, 255, 0.6)' : 'rgba(255, 255, 255, 0.15)',
747
- }}
748
- >
749
- <div style={{ display: 'flex', alignItems: 'center', gap: 8 }}>
750
- <Music size={14} style={{ opacity: 0.6 }} />
751
- <div>
752
- <div>{dance.label}</div>
753
- {dance.bgmUrl && (
754
- <div style={{ fontSize: 11, color: 'rgba(255,255,255,0.35)', marginTop: 1 }}>{t('含BGM', 'w/ BGM')}</div>
755
- )}
756
- </div>
757
- </div>
758
- <div
759
- onClick={(e) => {
760
- e.stopPropagation()
761
- fetch(`${FRIEND_API}/dance/delete`, {
762
- method: 'POST',
763
- headers: { 'Content-Type': 'application/json' },
764
- body: JSON.stringify({ id: dance.id }),
765
- })
766
- .then(() => {
767
- fetchCustomDances()
768
- if (currentDance === `custom:${dance.id}`) onDanceChange('love')
769
- })
770
- .catch(() => {})
771
- }}
772
- style={{
773
- width: 28,
774
- height: 28,
775
- borderRadius: 6,
776
- display: 'flex',
777
- alignItems: 'center',
778
- justifyContent: 'center',
779
- background: 'rgba(255, 255, 255, 0.08)',
780
- cursor: 'pointer',
781
- flexShrink: 0,
782
- opacity: 0.5,
783
- }}
784
- title={t('删除', 'Delete')}
785
- >
786
- <Trash2 size={13} />
787
- </div>
788
- </div>
789
- ))}
790
- </div>
791
- </div>
792
- )}
793
-
794
- <div style={{ marginTop: 12 }}>
795
- <div style={labelStyle}>{t('导入自定义舞蹈', 'Import Custom Dance')}</div>
796
- <button
797
- disabled={importingDance}
798
- onClick={async () => {
799
- setImportingDance(true)
800
- try {
801
- // Pick VMD file
802
- const vmdFile = await pickFile('.vmd')
803
- if (!vmdFile) { setImportingDance(false); return }
804
 
805
- // Import VMD
806
- const vmdFormData = new FormData()
807
- vmdFormData.append('file', vmdFile)
808
- const vmdRes = await fetch(`${FRIEND_API}/dance/import`, {
809
- method: 'POST',
810
- body: vmdFormData,
811
- })
812
- const vmdData = await vmdRes.json()
813
- if (!vmdData.ok) throw new Error(vmdData.error)
 
 
 
 
 
 
 
 
 
814
 
815
- // Ask for optional BGM
816
- const mp3File = await pickFile('.mp3')
817
- if (mp3File) {
818
- const mp3FormData = new FormData()
819
- mp3FormData.append('file', mp3File)
820
- await fetch(`${FRIEND_API}/dance/import`, {
821
- method: 'POST',
822
- body: mp3FormData,
823
- })
824
- }
 
 
 
 
 
 
 
 
 
 
 
 
825
 
826
- fetchCustomDances()
827
- } catch (err) {
828
- console.warn('Import dance failed:', err)
829
- }
830
- setImportingDance(false)
831
- }}
832
- style={{ ...applyBtnStyle, width: '100%', display: 'flex', alignItems: 'center', justifyContent: 'center', gap: 6 }}
833
- >
834
- {importingDance
835
- ? <Loader size={14} style={{ animation: 'spin 1s linear infinite' }} />
836
- : <Upload size={14} />}
837
- {importingDance ? t('导入中…', 'Importing…') : t('选择 VMD 舞蹈文件…', 'Select VMD dance file…')}
838
- </button>
839
- <div style={{ fontSize: 11, color: 'rgba(255,255,255,0.35)', marginTop: 4 }}>
840
- {t('选择 .vmd 舞蹈文件后,可选择配套 .mp3 音乐文件', 'Select a .vmd dance file, then optionally pick a matching .mp3')}
841
- </div>
842
- </div>
843
- </div>
844
- )}
845
- </div>
846
  </div>
847
  </div>
848
  )
@@ -1025,29 +655,3 @@ const selectStyle: React.CSSProperties = {
1025
  outline: 'none',
1026
  fontFamily: '"Segoe UI", "Microsoft YaHei", sans-serif',
1027
  }
1028
-
1029
- const textareaStyle: React.CSSProperties = {
1030
- width: '100%',
1031
- boxSizing: 'border-box',
1032
- border: '1px solid rgba(255, 255, 255, 0.2)',
1033
- borderRadius: 6,
1034
- background: 'rgba(0, 0, 0, 0.3)',
1035
- color: '#fff',
1036
- fontSize: 12,
1037
- lineHeight: '1.5',
1038
- padding: '6px 8px',
1039
- outline: 'none',
1040
- fontFamily: '"Segoe UI", "Microsoft YaHei", sans-serif',
1041
- resize: 'vertical',
1042
- }
1043
-
1044
- const applyBtnStyle: React.CSSProperties = {
1045
- height: 32,
1046
- padding: '0 12px',
1047
- border: 'none',
1048
- borderRadius: 6,
1049
- background: 'rgba(100, 160, 255, 0.5)',
1050
- color: '#fff',
1051
- fontSize: 13,
1052
- cursor: 'pointer',
1053
- }
 
1
  import { useState, useEffect, useRef, useCallback } from 'react'
2
+ import { X, Play, Loader } from 'lucide-react'
3
  import { FRIEND_API } from '../api'
 
 
 
 
 
 
 
 
 
 
4
 
5
  type SttProvider = 'browser' | 'groq' | 'anthropic' | 'local' | 'doubao'
6
 
 
21
  onVolumeChange: (v: number) => void
22
  uiAlign: 'left' | 'right'
23
  onUiAlignChange: (v: 'left' | 'right') => void
 
 
 
 
 
 
 
 
24
  language: 'zh' | 'en'
25
  onLanguageChange: (v: 'zh' | 'en') => void
 
 
26
  sttProvider?: SttProvider
27
  onSttProviderChange?: (v: SttProvider) => void
28
  }
29
 
30
+ type Tab = 'general' | 'voice' | 'model'
31
 
32
  const BUILTIN_MODELS = ['/friend/model1.vrm', '/friend/model2.vrm', '/friend/model3.vrm', '/friend/model4.vrm', '/friend/model5.vrm']
33
 
 
83
  tracking, onTrackingChange,
84
  volume, onVolumeChange,
85
  uiAlign, onUiAlignChange,
 
 
 
 
86
  language, onLanguageChange,
 
87
  sttProvider = 'browser', onSttProviderChange,
88
  }: SettingsPanelProps) {
89
  const t = (zh: string, en: string) => language === 'en' ? en : zh
90
 
91
  const [tab, setTab] = useState<Tab>('general')
 
 
 
 
 
 
92
  const [currentVoice, setCurrentVoice] = useState('')
93
  const [currentProvider, setCurrentProvider] = useState<string>('edge')
94
  const [qwenKey, setQwenKey] = useState('')
95
  const [qwenModel, setQwenModel] = useState('qwen3-tts-flash')
96
  const [previewingId, setPreviewingId] = useState<string | null>(null)
97
  const audioRef = useRef<HTMLAudioElement | null>(null)
 
 
 
98
 
99
  // Drag state
100
  const [panelPos, setPanelPos] = useState<{ x: number; y: number }>({ x: 0, y: 0 })
 
120
  window.addEventListener('mouseup', onUp)
121
  }, [panelPos])
122
 
 
123
  useEffect(() => {
124
  if (visible) setPanelPos({ x: 0, y: 0 })
125
  }, [visible])
126
 
127
  useEffect(() => {
128
  if (!visible) return
 
 
 
 
 
129
  fetch(`${FRIEND_API}/voice`)
130
  .then((r) => r.json())
131
  .then((data) => {
 
137
  .catch(() => {})
138
  }, [visible])
139
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
140
  const postVoiceSettings = (body: Record<string, string | undefined>) => {
141
  return fetch(`${FRIEND_API}/voice`, {
142
  method: 'POST',
 
180
  }, [])
181
 
182
  const preview = (voiceId: string) => {
 
183
  stopPreview()
184
  setPreviewingId(voiceId)
185
  fetch(`${FRIEND_API}/preview`, {
 
219
 
220
  {/* Tabs */}
221
  <div style={tabBarStyle}>
222
+ {(['general', 'voice', 'model'] as const).map((tb) => (
223
  <button
224
  key={tb}
225
  onClick={() => setTab(tb)}
226
  style={{ ...tabStyle, ...(tab === tb ? activeTabStyle : {}) }}
227
  >
228
+ {{ general: t('常规', 'General'), voice: t('语音', 'Voice'), model: t('形象', 'Model') }[tb]}
229
  </button>
230
  ))}
231
  </div>
 
234
  <div style={contentStyle}>
235
  {tab === 'general' && (
236
  <div style={{ display: 'flex', flexDirection: 'column', gap: 16 }}>
237
+ <LangToggle language={language} onChange={onLanguageChange} t={t} />
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
238
  <ToggleRow label={t('显示字幕', 'Subtitles')} value={showText} onChange={onShowTextChange} />
239
  <ToggleRow label={t('语音播报', 'TTS')} value={ttsEnabled} onChange={onTtsEnabledChange} />
240
+ <VolumeControl volume={volume} onChange={onVolumeChange} t={t} />
241
+ <TrackingControl tracking={tracking} onChange={onTrackingChange} t={t} />
242
+ <UIAlignControl uiAlign={uiAlign} onChange={onUiAlignChange} t={t} />
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
243
  <ToggleRow label={t('隐藏UI', 'Hide UI')} value={hideUI} onChange={onHideUIChange} />
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
244
  </div>
245
  )}
246
 
 
320
  </button>
321
  ))}
322
  </div>
 
 
 
 
 
323
  </div>
324
 
325
  <div style={{ marginTop: 8 }}>
 
357
  cursor: previewingId !== null ? 'default' : 'pointer',
358
  flexShrink: 0,
359
  opacity: previewingId !== null && previewingId !== v.id ? 0.3 : 0.7,
 
360
  }}
361
  title={t('试听', 'Preview')}
362
  >
 
382
  <option key={m} value={m}>{m.replace(/^\//, '')}</option>
383
  ))}
384
  </select>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
385
  </div>
386
  )}
387
+ </div>
388
+ </div>
389
+ </div>
390
+ )
391
+ }
392
 
393
+ function LangToggle({ language, onChange, t }: { language: 'zh' | 'en'; onChange: (v: 'zh' | 'en') => void; t: (zh: string, en: string) => string }) {
394
+ return (
395
+ <div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center' }}>
396
+ <span style={{ fontSize: 14 }}>{t('语言', 'Language')}</span>
397
+ <div style={{ display: 'flex', gap: 4 }}>
398
+ {(['zh', 'en'] as const).map((l) => (
399
+ <button
400
+ key={l}
401
+ onClick={() => onChange(l)}
402
+ style={{
403
+ ...smallBtnStyle,
404
+ background: language === l ? 'rgba(100, 160, 255, 0.4)' : 'rgba(255, 255, 255, 0.08)',
405
+ borderColor: language === l ? 'rgba(100, 160, 255, 0.6)' : 'rgba(255, 255, 255, 0.15)',
406
+ }}
407
+ >
408
+ {l === 'zh' ? '中文' : 'English'}
409
+ </button>
410
+ ))}
411
+ </div>
412
+ </div>
413
+ )
414
+ }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
415
 
416
+ function VolumeControl({ volume, onChange, t }: { volume: number; onChange: (v: number) => void; t: (zh: string, en: string) => string }) {
417
+ return (
418
+ <div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center' }}>
419
+ <span style={{ fontSize: 14 }}>{t('音量', 'Volume')}</span>
420
+ <div style={{ display: 'flex', alignItems: 'center', gap: 8 }}>
421
+ <input
422
+ type="range"
423
+ min={0}
424
+ max={100}
425
+ value={Math.round(volume * 100)}
426
+ onChange={(e) => onChange(Number(e.target.value) / 100)}
427
+ style={{ width: 100, accentColor: 'rgba(100, 160, 255, 0.8)' }}
428
+ />
429
+ <span style={{ fontSize: 12, color: 'rgba(255,255,255,0.5)', width: 28, textAlign: 'right' }}>{Math.round(volume * 100)}</span>
430
+ </div>
431
+ </div>
432
+ )
433
+ }
434
 
435
+ function TrackingControl({ tracking, onChange, t }: { tracking: 'mouse' | 'camera'; onChange: (v: 'mouse' | 'camera') => void; t: (zh: string, en: string) => string }) {
436
+ return (
437
+ <div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center' }}>
438
+ <span style={{ fontSize: 14 }}>{t('视线跟随', 'Eye Tracking')}</span>
439
+ <div style={{ display: 'flex', gap: 4 }}>
440
+ {(['mouse', 'camera'] as const).map((m) => (
441
+ <button
442
+ key={m}
443
+ onClick={() => onChange(m)}
444
+ style={{
445
+ ...smallBtnStyle,
446
+ background: tracking === m ? 'rgba(100, 160, 255, 0.4)' : 'rgba(255, 255, 255, 0.08)',
447
+ borderColor: tracking === m ? 'rgba(100, 160, 255, 0.6)' : 'rgba(255, 255, 255, 0.15)',
448
+ }}
449
+ >
450
+ {m === 'mouse' ? t('鼠标', 'Mouse') : t('镜头', 'Camera')}
451
+ </button>
452
+ ))}
453
+ </div>
454
+ </div>
455
+ )
456
+ }
457
 
458
+ function UIAlignControl({ uiAlign, onChange, t }: { uiAlign: 'left' | 'right'; onChange: (v: 'left' | 'right') => void; t: (zh: string, en: string) => string }) {
459
+ return (
460
+ <div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center' }}>
461
+ <span style={{ fontSize: 14 }}>{t('UI位置', 'UI Position')}</span>
462
+ <div style={{ display: 'flex', gap: 4 }}>
463
+ {(['left', 'right'] as const).map((a) => (
464
+ <button
465
+ key={a}
466
+ onClick={() => onChange(a)}
467
+ style={{
468
+ ...smallBtnStyle,
469
+ background: uiAlign === a ? 'rgba(100, 160, 255, 0.4)' : 'rgba(255, 255, 255, 0.08)',
470
+ borderColor: uiAlign === a ? 'rgba(100, 160, 255, 0.6)' : 'rgba(255, 255, 255, 0.15)',
471
+ }}
472
+ >
473
+ {a === 'left' ? t('靠左', 'Left') : t('靠右', 'Right')}
474
+ </button>
475
+ ))}
 
 
476
  </div>
477
  </div>
478
  )
 
655
  outline: 'none',
656
  fontFamily: '"Segoe UI", "Microsoft YaHei", sans-serif',
657
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
src/components/friend/frontend/components/TextBubble.tsx CHANGED
@@ -12,6 +12,7 @@ interface VrmMessage {
12
  audioUrl?: string
13
  audioIndex?: number
14
  imageUrl?: string
 
15
  sendFirstTts?: boolean
16
  appendText?: boolean
17
  replyDone?: boolean
 
12
  audioUrl?: string
13
  audioIndex?: number
14
  imageUrl?: string
15
+ action?: string
16
  sendFirstTts?: boolean
17
  appendText?: boolean
18
  replyDone?: boolean
src/components/friend/frontend/components/VRMScene.tsx CHANGED
@@ -1,4 +1,4 @@
1
- import { useEffect, useRef, useImperativeHandle, forwardRef, useState, useCallback } from 'react'
2
  import * as THREE from 'three'
3
  import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js'
4
  import { VRMLoaderPlugin, VRMUtils } from '@pixiv/three-vrm'
@@ -8,13 +8,9 @@ import { EmoteController } from '../emote'
8
  import { LipSync } from '../lip-sync'
9
  import { MotionController } from '../motion-controller'
10
 
11
- export type TouchRegion = 'head' | 'arm' | 'leg' | 'chest' | 'belly' | 'buttocks'
12
-
13
  interface VRMSceneProps {
14
  modelPath: string
15
  idleAnimationPath?: string
16
- onTouch?: (region: TouchRegion) => void
17
- onModelLoaded?: () => void
18
  }
19
 
20
  export type TrackingMode = 'mouse' | 'camera'
@@ -25,13 +21,8 @@ export interface VRMSceneHandle {
25
  resetCamera: () => void
26
  setTrackingMode: (mode: TrackingMode) => void
27
  playAction: (name: string, hold?: boolean) => void
28
- captureScreenshot: () => string | null
29
  panCamera: (dx: number, dy: number) => void
30
  rotateCamera: (dx: number, dy: number) => void
31
- playDance: (nameOrPreset: string | import('../motion-controller').DancePreset) => void
32
- stopDance: () => void
33
- isDancing: () => boolean
34
- setBgmVolume: (v: number) => void
35
  /** Unified reset: camera + resetToIdle + expressions to zero */
36
  reset: () => void
37
  }
@@ -42,6 +33,9 @@ interface BlinkState {
42
  blinkProgress: number
43
  timeSinceLastBlink: number
44
  nextBlinkTime: number
 
 
 
45
  }
46
 
47
  function createBlinkState(): BlinkState {
@@ -49,7 +43,10 @@ function createBlinkState(): BlinkState {
49
  isBlinking: false,
50
  blinkProgress: 0,
51
  timeSinceLastBlink: 0,
52
- nextBlinkTime: Math.random() * 4 + 1,
 
 
 
53
  }
54
  }
55
 
@@ -61,32 +58,37 @@ function updateBlink(vrm: VRM, delta: number, state: BlinkState) {
61
  if (!state.isBlinking && state.timeSinceLastBlink >= state.nextBlinkTime) {
62
  state.isBlinking = true
63
  state.blinkProgress = 0
 
 
 
 
64
  }
65
 
66
  if (state.isBlinking) {
67
- const BLINK_DURATION = 0.15
68
- state.blinkProgress += delta / BLINK_DURATION
69
  const blinkValue = Math.sin(Math.PI * state.blinkProgress)
70
  vrm.expressionManager.setValue('blink', blinkValue)
71
 
72
  if (state.blinkProgress >= 1) {
73
- state.isBlinking = false
74
- state.timeSinceLastBlink = 0
75
- vrm.expressionManager.setValue('blink', 0)
76
- state.nextBlinkTime = Math.random() * 5 + 1
 
 
 
 
 
 
 
 
 
 
77
  }
78
  }
79
  }
80
 
81
  // ── Relaxed hand pose ─────────────────────────────────────────────────────────
82
- // The idle_loop.vrma often has no finger tracks, so fingers stay in the stiff
83
- // T-pose. This must be applied EVERY FRAME after mixer.update() because the
84
- // AnimationMixer resets bones that have no tracks back to their rest rotation.
85
- //
86
- // VRM normalized bones use Z-axis for finger curl (spread is Y-axis).
87
- // Left hand curls positive Z, right hand curls negative Z.
88
- // We also add slight spread (Y-axis) variation per finger for a natural look,
89
- // and subtle per-frame micro-movement to avoid a "frozen" appearance.
90
 
91
  interface HandPoseCache {
92
  bones: { bone: THREE.Object3D; z: number; y: number }[]
@@ -101,7 +103,6 @@ function buildHandPoseCache(vrm: VRM): HandPoseCache {
101
  const segments = ['Proximal', 'Intermediate', 'Distal'] as const
102
  const sides = ['left', 'right'] as const
103
 
104
- // Base curl values — outer fingers curl more for a natural resting hand
105
  const curlMap: Record<string, [number, number, number]> = {
106
  Thumb: [0.25, 0.15, 0.10],
107
  Index: [0.20, 0.30, 0.20],
@@ -110,7 +111,6 @@ function buildHandPoseCache(vrm: VRM): HandPoseCache {
110
  Little: [0.35, 0.45, 0.30],
111
  }
112
 
113
- // Slight spread (Y-axis) to fan fingers apart naturally
114
  const spreadMap: Record<string, number> = {
115
  Thumb: 0.15,
116
  Index: 0.04,
@@ -132,7 +132,6 @@ function buildHandPoseCache(vrm: VRM): HandPoseCache {
132
  if (!bone) continue
133
 
134
  const z = sign * curls[s]
135
- // Only apply spread on the proximal segment
136
  const y = s === 0 ? sign * spread : 0
137
 
138
  bones.push({ bone, z, y })
@@ -144,32 +143,23 @@ function buildHandPoseCache(vrm: VRM): HandPoseCache {
144
  }
145
 
146
  function applyRelaxedHandPose(cache: HandPoseCache, time: number) {
 
 
 
 
147
  for (const { bone, z, y } of cache.bones) {
148
- // Subtle micro-movement: ±0.02 rad oscillation at slightly different
149
- // frequencies per bone (seeded by the base z value) to avoid uniformity
150
  const freq = 0.3 + Math.abs(z) * 2
151
  const micro = Math.sin(time * freq + z * 50) * 0.02
152
- bone.rotation.z = z + micro
153
  if (y !== 0) bone.rotation.y = y
154
  }
155
  }
156
 
157
-
158
  export const VRMScene = forwardRef<VRMSceneHandle, VRMSceneProps>(function VRMScene({
159
  modelPath,
160
  idleAnimationPath = '/friend/idle_loop.vrma',
161
- onTouch,
162
- onModelLoaded,
163
  }, ref) {
164
  const canvasRef = useRef<HTMLCanvasElement>(null)
165
- const [ripples, setRipples] = useState<{ id: number; x: number; y: number; confirmed: boolean }[]>([])
166
- const rippleIdRef = useRef(0)
167
-
168
- const spawnRipple = useCallback((x: number, y: number, confirmed: boolean) => {
169
- const id = ++rippleIdRef.current
170
- setRipples(prev => [...prev, { id, x, y, confirmed }])
171
- setTimeout(() => setRipples(prev => prev.filter(r => r.id !== id)), confirmed ? 700 : 500)
172
- }, [])
173
 
174
  const emoteRef = useRef<EmoteController | null>(null)
175
  const resetCameraRef = useRef<(() => void) | null>(null)
@@ -178,10 +168,6 @@ export const VRMScene = forwardRef<VRMSceneHandle, VRMSceneProps>(function VRMSc
178
  const panCameraRef = useRef<((dx: number, dy: number) => void) | null>(null)
179
  const rotateCameraRef = useRef<((dx: number, dy: number) => void) | null>(null)
180
  const lipSyncRef = useRef<LipSync>(LipSync.getInstance())
181
- const onTouchRef = useRef(onTouch)
182
- onTouchRef.current = onTouch
183
- const onModelLoadedRef = useRef(onModelLoaded)
184
- onModelLoadedRef.current = onModelLoaded
185
 
186
  useImperativeHandle(ref, () => ({
187
  setEmotion(emotion: string, intensity?: number) {
@@ -199,27 +185,12 @@ export const VRMScene = forwardRef<VRMSceneHandle, VRMSceneProps>(function VRMSc
199
  playAction(name: string, hold?: boolean) {
200
  motionRef.current?.playAction(name, hold)
201
  },
202
- captureScreenshot() {
203
- return canvasRef.current?.toDataURL('image/png') ?? null
204
- },
205
  panCamera(dx: number, dy: number) {
206
  panCameraRef.current?.(dx, dy)
207
  },
208
  rotateCamera(dx: number, dy: number) {
209
  rotateCameraRef.current?.(dx, dy)
210
  },
211
- playDance(nameOrPreset: string | import('../motion-controller').DancePreset) {
212
- motionRef.current?.playDance(nameOrPreset)
213
- },
214
- stopDance() {
215
- motionRef.current?.resetToIdle()
216
- },
217
- isDancing() {
218
- return motionRef.current?.isDancing ?? false
219
- },
220
- setBgmVolume(v: number) {
221
- motionRef.current?.setVolume(v)
222
- },
223
  reset() {
224
  resetCameraRef.current?.()
225
  motionRef.current?.resetToIdle()
@@ -253,12 +224,10 @@ export const VRMScene = forwardRef<VRMSceneHandle, VRMSceneProps>(function VRMSc
253
  0.1,
254
  100,
255
  )
256
- // Orbit state: camera orbits around the pivot point
257
- // Will be recalculated after model loads (like airi)
258
  const pivot = new THREE.Vector3(0, 0, 0)
259
  let orbitRadius = 2.0
260
- let orbitTheta = 0 // horizontal angle (radians)
261
- let orbitPhi = Math.PI / 2 // vertical angle (radians), PI/2 = eye level
262
 
263
  function updateCameraOrbit() {
264
  camera.position.set(
@@ -292,6 +261,18 @@ export const VRMScene = forwardRef<VRMSceneHandle, VRMSceneProps>(function VRMSc
292
  const saccades = new EyeSaccadeController()
293
  const lookAtTarget = { x: 0, y: 0, z: -100 }
294
 
 
 
 
 
 
 
 
 
 
 
 
 
295
  // ── Load VRM model, then load idle animation ─────────────────────────────
296
  loader.load(
297
  modelPath,
@@ -308,26 +289,24 @@ export const VRMScene = forwardRef<VRMSceneHandle, VRMSceneProps>(function VRMSc
308
  obj.frustumCulled = false
309
  })
310
 
311
- // Add lookAt quaternion proxy (from airi — needed for lookAt to work)
312
  if (loadedVrm.lookAt) {
313
  const lookAtQuatProxy = new VRMLookAtQuaternionProxy(loadedVrm.lookAt)
314
  lookAtQuatProxy.name = 'lookAtQuaternionProxy'
315
  loadedVrm.scene.add(lookAtQuatProxy)
316
  }
317
 
318
- // Normalize VRM 0.x to match 1.0 convention, then face camera
319
  VRMUtils.rotateVRM0(loadedVrm)
320
 
321
  scene.add(loadedVrm.scene)
322
  vrm = loadedVrm
323
 
324
- // ── Compute camera from model bounds (airi style) ───────────────────
325
  const box = new THREE.Box3().setFromObject(loadedVrm.scene)
326
  const modelSize = new THREE.Vector3()
327
  const modelCenter = new THREE.Vector3()
328
  box.getSize(modelSize)
329
  box.getCenter(modelCenter)
330
- modelCenter.y += modelSize.y / 3.2 // pivot at neck height
331
 
332
  const radians = (FOV / 2 * Math.PI) / 180
333
  const offsetX = modelSize.x / 16
@@ -340,7 +319,6 @@ export const VRMScene = forwardRef<VRMSceneHandle, VRMSceneProps>(function VRMSc
340
  orbitPhi = Math.PI / 2 - Math.atan2(offsetY, offsetZ)
341
  updateCameraOrbit()
342
 
343
- // Store initial state for reset
344
  const initPivot = pivot.clone()
345
  const initRadius = orbitRadius
346
  const initTheta = orbitTheta
@@ -357,8 +335,8 @@ export const VRMScene = forwardRef<VRMSceneHandle, VRMSceneProps>(function VRMSc
357
  const right = new THREE.Vector3()
358
  const up = new THREE.Vector3()
359
  camera.getWorldDirection(new THREE.Vector3())
360
- right.setFromMatrixColumn(camera.matrixWorld, 0) // camera right
361
- up.setFromMatrixColumn(camera.matrixWorld, 1) // camera up
362
  pivot.addScaledVector(right, -dx * 0.003)
363
  pivot.addScaledVector(up, dy * 0.003)
364
  updateCameraOrbit()
@@ -374,54 +352,27 @@ export const VRMScene = forwardRef<VRMSceneHandle, VRMSceneProps>(function VRMSc
374
  updateCameraOrbit()
375
  }
376
 
377
- // Build hand pose cache (applied every frame in animate loop)
378
  handPose = buildHandPoseCache(loadedVrm)
379
 
380
- // Initialize emote controller
 
 
 
 
 
 
 
381
  emote = new EmoteController(loadedVrm)
382
  emoteRef.current = emote
383
 
384
- // ── Initialize MotionController ──────────────────────────────────────
385
  motion = new MotionController(loadedVrm)
386
  motionRef.current = motion
387
 
388
- // Dance camera: auto-fit from hips position, centered on upper body
389
- const danceRadius = (modelSize.y / 1.6) / Math.tan((FOV / 2 * Math.PI) / 180)
390
-
391
- motion.onDanceStart = () => {
392
- const hipsNode = loadedVrm.humanoid?.getNormalizedBoneNode('hips')
393
- if (hipsNode) {
394
- const hipsWorld = new THREE.Vector3()
395
- hipsNode.getWorldPosition(hipsWorld)
396
- // Pivot at hips height (upper body center)
397
- pivot.set(hipsWorld.x, hipsWorld.y, hipsWorld.z)
398
- } else {
399
- pivot.copy(modelCenter)
400
- }
401
- orbitRadius = danceRadius
402
- orbitTheta = 0
403
- orbitPhi = Math.PI / 2
404
- updateCameraOrbit()
405
- }
406
- motion.onDanceStop = () => {
407
- pivot.copy(initPivot)
408
- orbitRadius = initRadius
409
- orbitTheta = initTheta
410
- orbitPhi = initPhi
411
- updateCameraOrbit()
412
- }
413
-
414
- // Load idle animation (non-blocking for fast startup)
415
  motion.loadIdle(idleAnimationPath).catch((err) =>
416
  console.warn('Failed to load idle animation:', err),
417
  )
418
 
419
- // Reset spring bones after everything is set up
420
  loadedVrm.springBoneManager?.reset()
421
-
422
- // Notify parent that model is ready (for auto-screenshot etc.)
423
- // Delay slightly so the first frame is rendered
424
- setTimeout(() => onModelLoadedRef.current?.(), 500)
425
  },
426
  () => {},
427
  (err) => {
@@ -431,7 +382,6 @@ export const VRMScene = forwardRef<VRMSceneHandle, VRMSceneProps>(function VRMSc
431
 
432
  // ── Mouse tracking ────────────────────────────────────────────────────────
433
  const mouse = new THREE.Vector2(0, 0)
434
-
435
  const _raycaster = new THREE.Raycaster()
436
  const _mouseVec = new THREE.Vector2()
437
 
@@ -441,7 +391,6 @@ export const VRMScene = forwardRef<VRMSceneHandle, VRMSceneProps>(function VRMSc
441
 
442
  if (trackingModeRef.current !== 'mouse') return
443
 
444
- // Compute lookAt target like airi's lookAtMouse
445
  _mouseVec.set(mouse.x, mouse.y)
446
  _raycaster.setFromCamera(_mouseVec, camera)
447
  const camDir = new THREE.Vector3()
@@ -461,7 +410,6 @@ export const VRMScene = forwardRef<VRMSceneHandle, VRMSceneProps>(function VRMSc
461
  }
462
  }
463
  }
464
- // Listen on both window and document to handle transparent window cases
465
  window.addEventListener('mousemove', onMouseMove)
466
  document.addEventListener('mousemove', onMouseMove)
467
 
@@ -481,92 +429,15 @@ export const VRMScene = forwardRef<VRMSceneHandle, VRMSceneProps>(function VRMSc
481
  }
482
  canvas.addEventListener('wheel', onWheel, { passive: false })
483
 
484
- // ── Drag controls ──────────────────────────────────────────────────────
485
- // Left drag: move window (startDragging)
486
- // Middle drag: dolly (zoom)
487
- // Right drag: rotate around model
488
- let dragMode: 'rotate' | 'dolly' | 'pan' | null = null
489
  let prevX = 0
490
  let prevY = 0
491
  const ROTATE_SPEED = 0.005
492
- const PAN_SPEED = 0.003
493
  const DOLLY_SPEED = 0.01
494
 
495
- // ── Touch interaction: detect body region from raycast hit ────────────
496
- const touchRaycaster = new THREE.Raycaster()
497
- const touchMouseVec = new THREE.Vector2()
498
- let lastTapTime = 0
499
- let lastTapRegion: TouchRegion | null = null
500
- let lastTouchFireTime = 0
501
- const DOUBLE_TAP_WINDOW = 500 // ms
502
- const TOUCH_COOLDOWN = 5_000 // ms
503
-
504
- // Bone-to-region mapping for proximity-based touch detection
505
- const boneRegionMap: [string, TouchRegion][] = [
506
- // Head
507
- ['head', 'head'], ['neck', 'head'],
508
- ['leftEye', 'head'], ['rightEye', 'head'], ['jaw', 'head'],
509
- // Arms
510
- ['leftShoulder', 'arm'], ['leftUpperArm', 'arm'], ['leftLowerArm', 'arm'], ['leftHand', 'arm'],
511
- ['rightShoulder', 'arm'], ['rightUpperArm', 'arm'], ['rightLowerArm', 'arm'], ['rightHand', 'arm'],
512
- // Legs
513
- ['leftUpperLeg', 'leg'], ['leftLowerLeg', 'leg'], ['leftFoot', 'leg'], ['leftToes', 'leg'],
514
- ['rightUpperLeg', 'leg'], ['rightLowerLeg', 'leg'], ['rightFoot', 'leg'], ['rightToes', 'leg'],
515
- // Torso
516
- ['chest', 'chest'], ['upperChest', 'chest'],
517
- ['spine', 'belly'],
518
- ['hips', 'buttocks'],
519
- ]
520
- const _bonePos = new THREE.Vector3()
521
-
522
- function detectTouchRegion(e: PointerEvent): TouchRegion | null {
523
- if (!vrm?.humanoid) return null
524
-
525
- touchMouseVec.set(
526
- (e.clientX / window.innerWidth) * 2 - 1,
527
- -(e.clientY / window.innerHeight) * 2 + 1,
528
- )
529
- touchRaycaster.setFromCamera(touchMouseVec, camera)
530
-
531
- const intersects = touchRaycaster.intersectObject(vrm.scene, true)
532
- if (intersects.length === 0) return null
533
-
534
- const hitPoint = intersects[0].point
535
-
536
- // Find closest bone to hit point
537
- let closestRegion: TouchRegion = 'belly'
538
- let closestDist = Infinity
539
-
540
- for (const [boneName, region] of boneRegionMap) {
541
- const bone = vrm.humanoid.getNormalizedBoneNode(boneName as any)
542
- if (!bone) continue
543
- bone.getWorldPosition(_bonePos)
544
- const dist = hitPoint.distanceToSquared(_bonePos)
545
- if (dist < closestDist) {
546
- closestDist = dist
547
- closestRegion = region
548
- }
549
- }
550
-
551
- return closestRegion
552
- }
553
-
554
- // ── Left-click: distinguish click (touch) vs drag (move window) ────
555
- let leftDownPos: { x: number; y: number; time: number; region: TouchRegion | null } | null = null
556
- const CLICK_MOVE_THRESHOLD = 5 // px
557
- const CLICK_TIME_THRESHOLD = 300 // ms
558
-
559
  function onPointerDown(e: PointerEvent) {
560
- if (e.button === 0) {
561
- const region = detectTouchRegion(e)
562
- if (region) {
563
- // Might be a touch — wait for pointerup to confirm it's not a drag
564
- leftDownPos = { x: e.clientX, y: e.clientY, time: Date.now(), region }
565
- return
566
- }
567
- // Not on model — no window drag available in web mode
568
- return
569
- } else if (e.button === 1) {
570
  dragMode = 'dolly'
571
  e.preventDefault()
572
  } else if (e.button === 2) {
@@ -580,15 +451,6 @@ export const VRMScene = forwardRef<VRMSceneHandle, VRMSceneProps>(function VRMSc
580
  }
581
 
582
  function onPointerMove(e: PointerEvent) {
583
- // If left button held on model and moved beyond threshold → it's a drag, not a touch
584
- if (leftDownPos && e.buttons & 1) {
585
- const dx = Math.abs(e.clientX - leftDownPos.x)
586
- const dy = Math.abs(e.clientY - leftDownPos.y)
587
- if (dx > CLICK_MOVE_THRESHOLD || dy > CLICK_MOVE_THRESHOLD) {
588
- leftDownPos = null
589
- return
590
- }
591
- }
592
  if (!dragMode) return
593
  const dx = e.clientX - prevX
594
  const dy = e.clientY - prevY
@@ -608,42 +470,13 @@ export const VRMScene = forwardRef<VRMSceneHandle, VRMSceneProps>(function VRMSc
608
  MIN_RADIUS,
609
  MAX_RADIUS,
610
  )
611
- } else if (dragMode === 'pan') {
612
- pivot.x -= dx * PAN_SPEED
613
- pivot.y += dy * PAN_SPEED
614
  }
615
  updateCameraOrbit()
616
  }
617
 
618
- function onPointerUp(e: PointerEvent) {
619
- // Confirm touch: short press with no movement on model
620
- if (leftDownPos && e.button === 0) {
621
- const elapsed = Date.now() - leftDownPos.time
622
- const dx = Math.abs(e.clientX - leftDownPos.x)
623
- const dy = Math.abs(e.clientY - leftDownPos.y)
624
- if (elapsed < CLICK_TIME_THRESHOLD && dx <= CLICK_MOVE_THRESHOLD && dy <= CLICK_MOVE_THRESHOLD) {
625
- const now = Date.now()
626
- const region = leftDownPos.region!
627
- if (now - lastTapTime < DOUBLE_TAP_WINDOW && lastTapRegion === region && now - lastTouchFireTime > TOUCH_COOLDOWN) {
628
- // Double-tap confirmed
629
- lastTouchFireTime = now
630
- spawnRipple(e.clientX, e.clientY, true)
631
- onTouchRef.current?.(region)
632
- lastTapTime = 0
633
- lastTapRegion = null
634
- } else {
635
- // First tap — wait for second
636
- spawnRipple(e.clientX, e.clientY, false)
637
- lastTapTime = now
638
- lastTapRegion = region
639
- }
640
- }
641
- leftDownPos = null
642
- return
643
- }
644
  if (dragMode) {
645
  dragMode = null
646
- canvas!.releasePointerCapture(e.pointerId)
647
  }
648
  }
649
 
@@ -664,20 +497,6 @@ export const VRMScene = forwardRef<VRMSceneHandle, VRMSceneProps>(function VRMSc
664
  }
665
  window.addEventListener('resize', onResize)
666
 
667
- // ── Hit-test for window pass-through ──────────────────────────────────────
668
- // Offscreen render target: render scene, read 1 pixel alpha at cursor.
669
- // Updated in the render loop — no extra render pass, just piggybacks.
670
- const hitTarget = new THREE.WebGLRenderTarget(1, 1, { depthBuffer: false })
671
- let pendingHitTest: { x: number; y: number; resolve: (hit: boolean) => void } | null = null
672
- const hitPixel = new Uint8Array(4)
673
-
674
- // Async hit-test: queues a request, resolved after next frame render
675
- ;(window as any).__clawHitTest = (clientX: number, clientY: number): Promise<boolean> => {
676
- return new Promise((resolve) => {
677
- pendingHitTest = { x: clientX, y: clientY, resolve }
678
- })
679
- }
680
-
681
  // ── Animation loop ────────────────────────────────────────────────────────
682
  const clock = new THREE.Clock()
683
  let animFrameId: number
@@ -687,16 +506,12 @@ export const VRMScene = forwardRef<VRMSceneHandle, VRMSceneProps>(function VRMSc
687
  const delta = clock.getDelta()
688
 
689
  if (vrm) {
690
- // 1. Animation mixer
691
  motion?.update(delta)
692
 
693
- // 1.5. Relaxed hand pose — skip during dance (VMD has own hand anim)
694
- if (handPose && !motion?.isDancing) applyRelaxedHandPose(handPose, clock.elapsedTime)
695
 
696
- // 2. Humanoid update
697
  vrm.humanoid?.update()
698
 
699
- // 3. Camera tracking mode: look at camera position
700
  if (trackingModeRef.current === 'camera') {
701
  lookAtTarget.x = camera.position.x
702
  lookAtTarget.y = camera.position.y
@@ -704,60 +519,84 @@ export const VRMScene = forwardRef<VRMSceneHandle, VRMSceneProps>(function VRMSc
704
  saccades.instantUpdate(vrm, lookAtTarget)
705
  }
706
 
707
- // 4. LookAt update
708
  vrm.lookAt?.update(delta)
709
-
710
- // 5. Eye saccades (airi style)
711
  saccades.update(vrm, lookAtTarget, delta)
712
-
713
- // 5. Blinking
714
  updateBlink(vrm, delta, blinkState)
715
-
716
- // 6. Emote transitions
717
  emote?.update(delta)
718
-
719
- // 7. Lip sync
720
  lipSyncRef.current.update(vrm, delta)
721
 
722
- // 8. Expression manager (apply blink etc.)
723
- vrm.expressionManager?.update()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
724
 
725
- // 8. Spring bone physics
726
  vrm.springBoneManager?.update(delta)
727
- }
728
-
729
- renderer.render(scene, camera)
730
 
731
- // Process pending hit-test after render
732
- if (pendingHitTest && canvas) {
733
- const { x, y, resolve } = pendingHitTest
734
- pendingHitTest = null
735
-
736
- if (!vrm) {
737
- resolve(true) // Model not loaded — don't pass through
738
- } else {
739
 
740
- const dpr = renderer.getPixelRatio()
741
- const bufW = canvas.clientWidth * dpr
742
- const bufH = canvas.clientHeight * dpr
 
 
 
743
 
744
- if (hitTarget.width !== bufW || hitTarget.height !== bufH) {
745
- hitTarget.setSize(bufW, bufH)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
746
  }
747
 
748
- // Render to offscreen target
749
- renderer.setRenderTarget(hitTarget)
750
- renderer.clear()
751
- renderer.render(scene, camera)
752
- // Read 1 pixel at cursor position
753
- const px = Math.floor(x * dpr)
754
- const py = Math.floor(bufH - y * dpr) // GL Y-flip
755
- renderer.readRenderTargetPixels(hitTarget, px, py, 1, 1, hitPixel)
756
- renderer.setRenderTarget(null)
757
-
758
- resolve(hitPixel[3] > 10)
759
- } // end else (vrm exists)
760
  }
 
 
761
  }
762
 
763
  animate()
@@ -777,7 +616,6 @@ export const VRMScene = forwardRef<VRMSceneHandle, VRMSceneProps>(function VRMSc
777
  emoteRef.current = null
778
  motion?.dispose()
779
  motionRef.current = null
780
- hitTarget.dispose()
781
  delete (window as any).__clawHitTest
782
  renderer.dispose()
783
  }
@@ -795,18 +633,11 @@ export const VRMScene = forwardRef<VRMSceneHandle, VRMSceneProps>(function VRMSc
795
  cursor: 'grab',
796
  }}
797
  />
798
- {ripples.map(r => (
799
- <span
800
- key={r.id}
801
- className={r.confirmed ? 'touch-ripple confirmed' : 'touch-ripple'}
802
- style={{ left: r.x, top: r.y }}
803
- />
804
- ))}
805
  </div>
806
  )
807
  })
808
 
809
- // ── Eye saccade interval (from airi) ─────────────────────────────────────────
810
  const EYE_SACCADE_INT_STEP = 400
811
  const EYE_SACCADE_INT_P: number[][] = [
812
  [0.075, 800], [0.110, 0], [0.125, 0], [0.140, 0], [0.125, 0],
@@ -827,13 +658,11 @@ function randomSaccadeInterval(): number {
827
  return EYE_SACCADE_INT_P[EYE_SACCADE_INT_P.length - 1][1] + Math.random() * EYE_SACCADE_INT_STEP
828
  }
829
 
830
- // ── Eye saccade controller (from airi) ───────────────────────────────────────
831
  class EyeSaccadeController {
832
  private nextSaccadeAfter = -1
833
  private timeSinceLastSaccade = 0
834
  private fixationTarget = new THREE.Vector3()
835
 
836
- /** Called when lookAt target changes (e.g. mouse moved) */
837
  instantUpdate(vrm: VRM, target: { x: number; y: number; z: number }) {
838
  this.fixationTarget.set(target.x, target.y, target.z)
839
  if (!vrm.lookAt) return
@@ -844,12 +673,10 @@ class EyeSaccadeController {
844
  vrm.lookAt.update(0.016)
845
  }
846
 
847
- /** Called every frame */
848
  update(vrm: VRM, lookAtTarget: { x: number; y: number; z: number }, delta: number) {
849
  if (!vrm.expressionManager || !vrm.lookAt) return
850
 
851
  if (this.timeSinceLastSaccade >= this.nextSaccadeAfter) {
852
- // Add random offset to the current lookAt target
853
  this.fixationTarget.set(
854
  lookAtTarget.x + THREE.MathUtils.randFloat(-0.25, 0.25),
855
  lookAtTarget.y + THREE.MathUtils.randFloat(-0.25, 0.25),
 
1
+ import { useEffect, useRef, useImperativeHandle, forwardRef } from 'react'
2
  import * as THREE from 'three'
3
  import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js'
4
  import { VRMLoaderPlugin, VRMUtils } from '@pixiv/three-vrm'
 
8
  import { LipSync } from '../lip-sync'
9
  import { MotionController } from '../motion-controller'
10
 
 
 
11
  interface VRMSceneProps {
12
  modelPath: string
13
  idleAnimationPath?: string
 
 
14
  }
15
 
16
  export type TrackingMode = 'mouse' | 'camera'
 
21
  resetCamera: () => void
22
  setTrackingMode: (mode: TrackingMode) => void
23
  playAction: (name: string, hold?: boolean) => void
 
24
  panCamera: (dx: number, dy: number) => void
25
  rotateCamera: (dx: number, dy: number) => void
 
 
 
 
26
  /** Unified reset: camera + resetToIdle + expressions to zero */
27
  reset: () => void
28
  }
 
33
  blinkProgress: number
34
  timeSinceLastBlink: number
35
  nextBlinkTime: number
36
+ blinkDuration: number
37
+ doubleBlink: boolean
38
+ doubleBlinkCount: number
39
  }
40
 
41
  function createBlinkState(): BlinkState {
 
43
  isBlinking: false,
44
  blinkProgress: 0,
45
  timeSinceLastBlink: 0,
46
+ nextBlinkTime: Math.random() * 5 + 2,
47
+ blinkDuration: 0.1 + Math.random() * 0.08,
48
+ doubleBlink: false,
49
+ doubleBlinkCount: 0,
50
  }
51
  }
52
 
 
58
  if (!state.isBlinking && state.timeSinceLastBlink >= state.nextBlinkTime) {
59
  state.isBlinking = true
60
  state.blinkProgress = 0
61
+ state.blinkDuration = 0.08 + Math.random() * 0.12
62
+ // ~8% chance of double blink (more natural)
63
+ state.doubleBlink = Math.random() < 0.08
64
+ state.doubleBlinkCount = 0
65
  }
66
 
67
  if (state.isBlinking) {
68
+ state.blinkProgress += delta / state.blinkDuration
 
69
  const blinkValue = Math.sin(Math.PI * state.blinkProgress)
70
  vrm.expressionManager.setValue('blink', blinkValue)
71
 
72
  if (state.blinkProgress >= 1) {
73
+ state.blinkProgress = 0
74
+ state.doubleBlinkCount++
75
+
76
+ if (state.doubleBlink && state.doubleBlinkCount < 2) {
77
+ // Quick reopen then re-blink
78
+ vrm.expressionManager.setValue('blink', 0)
79
+ state.blinkDuration = 0.06 + Math.random() * 0.06
80
+ } else {
81
+ state.isBlinking = false
82
+ state.timeSinceLastBlink = 0
83
+ vrm.expressionManager.setValue('blink', 0)
84
+ state.doubleBlink = false
85
+ state.nextBlinkTime = Math.random() * 6 + 1.5
86
+ }
87
  }
88
  }
89
  }
90
 
91
  // ── Relaxed hand pose ─────────────────────────────────────────────────────────
 
 
 
 
 
 
 
 
92
 
93
  interface HandPoseCache {
94
  bones: { bone: THREE.Object3D; z: number; y: number }[]
 
103
  const segments = ['Proximal', 'Intermediate', 'Distal'] as const
104
  const sides = ['left', 'right'] as const
105
 
 
106
  const curlMap: Record<string, [number, number, number]> = {
107
  Thumb: [0.25, 0.15, 0.10],
108
  Index: [0.20, 0.30, 0.20],
 
111
  Little: [0.35, 0.45, 0.30],
112
  }
113
 
 
114
  const spreadMap: Record<string, number> = {
115
  Thumb: 0.15,
116
  Index: 0.04,
 
132
  if (!bone) continue
133
 
134
  const z = sign * curls[s]
 
135
  const y = s === 0 ? sign * spread : 0
136
 
137
  bones.push({ bone, z, y })
 
143
  }
144
 
145
  function applyRelaxedHandPose(cache: HandPoseCache, time: number) {
146
+ // Slow hand tension cycle (~30s period): hands subtly change posture over time
147
+ const tensionCycle = 0.5 + 0.5 * Math.sin(time * 0.035)
148
+ const zScale = 0.85 + tensionCycle * 0.3 // 0.85-1.15
149
+
150
  for (const { bone, z, y } of cache.bones) {
 
 
151
  const freq = 0.3 + Math.abs(z) * 2
152
  const micro = Math.sin(time * freq + z * 50) * 0.02
153
+ bone.rotation.z = z * zScale + micro
154
  if (y !== 0) bone.rotation.y = y
155
  }
156
  }
157
 
 
158
  export const VRMScene = forwardRef<VRMSceneHandle, VRMSceneProps>(function VRMScene({
159
  modelPath,
160
  idleAnimationPath = '/friend/idle_loop.vrma',
 
 
161
  }, ref) {
162
  const canvasRef = useRef<HTMLCanvasElement>(null)
 
 
 
 
 
 
 
 
163
 
164
  const emoteRef = useRef<EmoteController | null>(null)
165
  const resetCameraRef = useRef<(() => void) | null>(null)
 
168
  const panCameraRef = useRef<((dx: number, dy: number) => void) | null>(null)
169
  const rotateCameraRef = useRef<((dx: number, dy: number) => void) | null>(null)
170
  const lipSyncRef = useRef<LipSync>(LipSync.getInstance())
 
 
 
 
171
 
172
  useImperativeHandle(ref, () => ({
173
  setEmotion(emotion: string, intensity?: number) {
 
185
  playAction(name: string, hold?: boolean) {
186
  motionRef.current?.playAction(name, hold)
187
  },
 
 
 
188
  panCamera(dx: number, dy: number) {
189
  panCameraRef.current?.(dx, dy)
190
  },
191
  rotateCamera(dx: number, dy: number) {
192
  rotateCameraRef.current?.(dx, dy)
193
  },
 
 
 
 
 
 
 
 
 
 
 
 
194
  reset() {
195
  resetCameraRef.current?.()
196
  motionRef.current?.resetToIdle()
 
224
  0.1,
225
  100,
226
  )
 
 
227
  const pivot = new THREE.Vector3(0, 0, 0)
228
  let orbitRadius = 2.0
229
+ let orbitTheta = 0
230
+ let orbitPhi = Math.PI / 2
231
 
232
  function updateCameraOrbit() {
233
  camera.position.set(
 
261
  const saccades = new EyeSaccadeController()
262
  const lookAtTarget = { x: 0, y: 0, z: -100 }
263
 
264
+ // ── Aliveness system state: breathing, sway, speech micro-movements ─────
265
+ let alivenessBones = { chestBone: null as THREE.Object3D | null, spineBone: null as THREE.Object3D | null, neckBone: null as THREE.Object3D | null }
266
+ let breathPhase = Math.random() * Math.PI * 2
267
+ let swayPhase = Math.random() * Math.PI * 2
268
+ let speechBlend = 0
269
+ let userListenBlend = 0
270
+ // Micro-expression state
271
+ let microTimer = 5 + Math.random() * 10
272
+ let microActive = false
273
+ let microPhase = 0
274
+ let microShape = ''
275
+
276
  // ── Load VRM model, then load idle animation ─────────────────────────────
277
  loader.load(
278
  modelPath,
 
289
  obj.frustumCulled = false
290
  })
291
 
 
292
  if (loadedVrm.lookAt) {
293
  const lookAtQuatProxy = new VRMLookAtQuaternionProxy(loadedVrm.lookAt)
294
  lookAtQuatProxy.name = 'lookAtQuaternionProxy'
295
  loadedVrm.scene.add(lookAtQuatProxy)
296
  }
297
 
 
298
  VRMUtils.rotateVRM0(loadedVrm)
299
 
300
  scene.add(loadedVrm.scene)
301
  vrm = loadedVrm
302
 
303
+ // ── Compute camera from model bounds ───────────────────
304
  const box = new THREE.Box3().setFromObject(loadedVrm.scene)
305
  const modelSize = new THREE.Vector3()
306
  const modelCenter = new THREE.Vector3()
307
  box.getSize(modelSize)
308
  box.getCenter(modelCenter)
309
+ modelCenter.y += modelSize.y / 3.2
310
 
311
  const radians = (FOV / 2 * Math.PI) / 180
312
  const offsetX = modelSize.x / 16
 
319
  orbitPhi = Math.PI / 2 - Math.atan2(offsetY, offsetZ)
320
  updateCameraOrbit()
321
 
 
322
  const initPivot = pivot.clone()
323
  const initRadius = orbitRadius
324
  const initTheta = orbitTheta
 
335
  const right = new THREE.Vector3()
336
  const up = new THREE.Vector3()
337
  camera.getWorldDirection(new THREE.Vector3())
338
+ right.setFromMatrixColumn(camera.matrixWorld, 0)
339
+ up.setFromMatrixColumn(camera.matrixWorld, 1)
340
  pivot.addScaledVector(right, -dx * 0.003)
341
  pivot.addScaledVector(up, dy * 0.003)
342
  updateCameraOrbit()
 
352
  updateCameraOrbit()
353
  }
354
 
 
355
  handPose = buildHandPoseCache(loadedVrm)
356
 
357
+ // ── Initialize aliveness bone references ──
358
+ const h = loadedVrm.humanoid
359
+ alivenessBones = {
360
+ chestBone: h?.getNormalizedBoneNode('chest') ?? null,
361
+ spineBone: h?.getNormalizedBoneNode('spine') ?? null,
362
+ neckBone: h?.getNormalizedBoneNode('neck') ?? null,
363
+ }
364
+
365
  emote = new EmoteController(loadedVrm)
366
  emoteRef.current = emote
367
 
 
368
  motion = new MotionController(loadedVrm)
369
  motionRef.current = motion
370
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
371
  motion.loadIdle(idleAnimationPath).catch((err) =>
372
  console.warn('Failed to load idle animation:', err),
373
  )
374
 
 
375
  loadedVrm.springBoneManager?.reset()
 
 
 
 
376
  },
377
  () => {},
378
  (err) => {
 
382
 
383
  // ── Mouse tracking ────────────────────────────────────────────────────────
384
  const mouse = new THREE.Vector2(0, 0)
 
385
  const _raycaster = new THREE.Raycaster()
386
  const _mouseVec = new THREE.Vector2()
387
 
 
391
 
392
  if (trackingModeRef.current !== 'mouse') return
393
 
 
394
  _mouseVec.set(mouse.x, mouse.y)
395
  _raycaster.setFromCamera(_mouseVec, camera)
396
  const camDir = new THREE.Vector3()
 
410
  }
411
  }
412
  }
 
413
  window.addEventListener('mousemove', onMouseMove)
414
  document.addEventListener('mousemove', onMouseMove)
415
 
 
429
  }
430
  canvas.addEventListener('wheel', onWheel, { passive: false })
431
 
432
+ // ── Camera drag controls ──────────────────────────────────────────────
433
+ let dragMode: 'rotate' | 'dolly' | null = null
 
 
 
434
  let prevX = 0
435
  let prevY = 0
436
  const ROTATE_SPEED = 0.005
 
437
  const DOLLY_SPEED = 0.01
438
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
439
  function onPointerDown(e: PointerEvent) {
440
+ if (e.button === 1) {
 
 
 
 
 
 
 
 
 
441
  dragMode = 'dolly'
442
  e.preventDefault()
443
  } else if (e.button === 2) {
 
451
  }
452
 
453
  function onPointerMove(e: PointerEvent) {
 
 
 
 
 
 
 
 
 
454
  if (!dragMode) return
455
  const dx = e.clientX - prevX
456
  const dy = e.clientY - prevY
 
470
  MIN_RADIUS,
471
  MAX_RADIUS,
472
  )
 
 
 
473
  }
474
  updateCameraOrbit()
475
  }
476
 
477
+ function onPointerUp() {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
478
  if (dragMode) {
479
  dragMode = null
 
480
  }
481
  }
482
 
 
497
  }
498
  window.addEventListener('resize', onResize)
499
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
500
  // ── Animation loop ────────────────────────────────────────────────────────
501
  const clock = new THREE.Clock()
502
  let animFrameId: number
 
506
  const delta = clock.getDelta()
507
 
508
  if (vrm) {
 
509
  motion?.update(delta)
510
 
511
+ if (handPose) applyRelaxedHandPose(handPose, clock.elapsedTime)
 
512
 
 
513
  vrm.humanoid?.update()
514
 
 
515
  if (trackingModeRef.current === 'camera') {
516
  lookAtTarget.x = camera.position.x
517
  lookAtTarget.y = camera.position.y
 
519
  saccades.instantUpdate(vrm, lookAtTarget)
520
  }
521
 
 
522
  vrm.lookAt?.update(delta)
 
 
523
  saccades.update(vrm, lookAtTarget, delta)
 
 
524
  updateBlink(vrm, delta, blinkState)
 
 
525
  emote?.update(delta)
 
 
526
  lipSyncRef.current.update(vrm, delta)
527
 
528
+ // ── Aliveness: micro-expressions (asymmetric blink, subtle morphs) ──
529
+ microTimer -= delta
530
+ if (microTimer <= 0 && !microActive) {
531
+ microShape = Math.random() > 0.5 ? 'blinkLeft' : 'blinkRight'
532
+ microActive = true
533
+ microPhase = 0
534
+ }
535
+ if (microActive && vrm.expressionManager) {
536
+ microPhase += delta * 4
537
+ const val = Math.sin(Math.PI * Math.min(microPhase, 1))
538
+ vrm.expressionManager.setValue(microShape, val * 0.3)
539
+ if (microPhase >= 2) {
540
+ microActive = false
541
+ vrm.expressionManager.setValue(microShape, 0)
542
+ microTimer = 8 + Math.random() * 16
543
+ }
544
+ }
545
 
546
+ vrm.expressionManager?.update()
547
  vrm.springBoneManager?.update(delta)
 
 
 
548
 
549
+ // ── Aliveness: breathing (chest rise/fall, post-mixer) ──
550
+ // Breathing rate varies naturally, amplitude tuned for VRM scale
551
+ const breathRate = 1.8 + Math.sin(breathPhase * 0.05) * 0.4 + Math.sin(clock.elapsedTime * 0.1) * 0.3
552
+ breathPhase += delta * breathRate
553
+ const breathVal = Math.sin(breathPhase) * 0.006
554
+ if (alivenessBones.chestBone) {
555
+ alivenessBones.chestBone.position.y += breathVal
556
+ }
557
 
558
+ // ── Aliveness: postural sway ──
559
+ swayPhase += delta * 0.35
560
+ const swayZ = Math.sin(swayPhase) * 0.005
561
+ if (alivenessBones.spineBone) {
562
+ alivenessBones.spineBone.rotation.z += swayZ
563
+ }
564
 
565
+ // ── Aliveness: speech-driven micro-movements ──
566
+ const isSpeaking = lipSyncRef.current.isActive()
567
+ const targetBlend = isSpeaking ? 1 : 0
568
+ speechBlend += (targetBlend - speechBlend) * Math.min(1, delta * 3)
569
+ if (speechBlend > 0.01) {
570
+ const t = clock.elapsedTime
571
+ const headX = Math.sin(t * 3.7 + 1.2) * 0.02 * speechBlend
572
+ const headZ = Math.sin(t * 2.3 + 0.7) * 0.015 * speechBlend
573
+ const headY = Math.sin(t * 1.5 + 3.8) * 0.008 * speechBlend // slight rotation (looking around while talking)
574
+ const spineSway = Math.sin(t * 1.8 + 0.3) * 0.006 * speechBlend
575
+ if (alivenessBones.neckBone) {
576
+ alivenessBones.neckBone.rotation.x += headX
577
+ alivenessBones.neckBone.rotation.z += headZ
578
+ alivenessBones.neckBone.rotation.y += headY
579
+ }
580
+ if (alivenessBones.spineBone) {
581
+ alivenessBones.spineBone.rotation.x += spineSway
582
+ }
583
  }
584
 
585
+ // ── Aliveness: listening response when user is speaking ──
586
+ const isUserSpeaking = !!(window as any).__userRecording
587
+ userListenBlend += ((isUserSpeaking ? 1 : 0) - userListenBlend) * Math.min(1, delta * 2)
588
+ if (userListenBlend > 0.01) {
589
+ const t = clock.elapsedTime
590
+ const listenTilt = Math.sin(t * 0.9 + 0.5) * 0.012 * userListenBlend // head tilt
591
+ const listenNod = Math.sin(t * 2.3 + 3.1) * 0.008 * userListenBlend // subtle nodding
592
+ if (alivenessBones.neckBone) {
593
+ alivenessBones.neckBone.rotation.z += listenTilt
594
+ alivenessBones.neckBone.rotation.x += listenNod
595
+ }
596
+ }
597
  }
598
+
599
+ renderer.render(scene, camera)
600
  }
601
 
602
  animate()
 
616
  emoteRef.current = null
617
  motion?.dispose()
618
  motionRef.current = null
 
619
  delete (window as any).__clawHitTest
620
  renderer.dispose()
621
  }
 
633
  cursor: 'grab',
634
  }}
635
  />
 
 
 
 
 
 
 
636
  </div>
637
  )
638
  })
639
 
640
+ // ── Eye saccade interval ─────────────────────────────────────────
641
  const EYE_SACCADE_INT_STEP = 400
642
  const EYE_SACCADE_INT_P: number[][] = [
643
  [0.075, 800], [0.110, 0], [0.125, 0], [0.140, 0], [0.125, 0],
 
658
  return EYE_SACCADE_INT_P[EYE_SACCADE_INT_P.length - 1][1] + Math.random() * EYE_SACCADE_INT_STEP
659
  }
660
 
 
661
  class EyeSaccadeController {
662
  private nextSaccadeAfter = -1
663
  private timeSinceLastSaccade = 0
664
  private fixationTarget = new THREE.Vector3()
665
 
 
666
  instantUpdate(vrm: VRM, target: { x: number; y: number; z: number }) {
667
  this.fixationTarget.set(target.x, target.y, target.z)
668
  if (!vrm.lookAt) return
 
673
  vrm.lookAt.update(0.016)
674
  }
675
 
 
676
  update(vrm: VRM, lookAtTarget: { x: number; y: number; z: number }, delta: number) {
677
  if (!vrm.expressionManager || !vrm.lookAt) return
678
 
679
  if (this.timeSinceLastSaccade >= this.nextSaccadeAfter) {
 
680
  this.fixationTarget.set(
681
  lookAtTarget.x + THREE.MathUtils.randFloat(-0.25, 0.25),
682
  lookAtTarget.y + THREE.MathUtils.randFloat(-0.25, 0.25),
src/components/friend/frontend/index.html CHANGED
@@ -52,45 +52,6 @@
52
  100% { opacity: 1; transform: scale(1) translateY(0); }
53
  }
54
 
55
- @keyframes mood-bubble-float {
56
- 0% { opacity: 1; transform: translateX(-50%) translateY(0) scale(0.8); }
57
- 15% { opacity: 1; transform: translateX(-50%) translateY(-8px) scale(1.1); }
58
- 30% { opacity: 1; transform: translateX(-50%) translateY(-16px) scale(1); }
59
- 70% { opacity: 0.9; }
60
- 100% { opacity: 0; transform: translateX(-50%) translateY(-50px) scale(0.9); }
61
- }
62
-
63
- /* Touch ripple */
64
- .touch-ripple {
65
- position: fixed;
66
- pointer-events: none;
67
- width: 40px;
68
- height: 40px;
69
- margin-left: -20px;
70
- margin-top: -20px;
71
- border-radius: 50%;
72
- border: 2px solid rgba(255, 255, 255, 0.5);
73
- animation: ripple-expand 0.5s ease-out forwards;
74
- }
75
- .touch-ripple.confirmed {
76
- width: 60px;
77
- height: 60px;
78
- margin-left: -30px;
79
- margin-top: -30px;
80
- border: 2.5px solid rgba(255, 180, 220, 0.8);
81
- box-shadow: 0 0 12px rgba(255, 180, 220, 0.4);
82
- animation: ripple-expand-confirmed 0.7s ease-out forwards;
83
- }
84
-
85
- @keyframes ripple-expand {
86
- 0% { transform: scale(0); opacity: 0.7; }
87
- 100% { transform: scale(1.8); opacity: 0; }
88
- }
89
- @keyframes ripple-expand-confirmed {
90
- 0% { transform: scale(0); opacity: 0.9; }
91
- 50% { transform: scale(1.2); opacity: 0.6; }
92
- 100% { transform: scale(2.2); opacity: 0; }
93
- }
94
  </style>
95
  </head>
96
 
 
52
  100% { opacity: 1; transform: scale(1) translateY(0); }
53
  }
54
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
55
  </style>
56
  </head>
57
 
src/components/friend/frontend/lip-sync.ts CHANGED
@@ -167,6 +167,11 @@ export class LipSync {
167
  }
168
  }
169
 
 
 
 
 
 
170
  /** Stop current audio playback immediately. */
171
  stopAudio() {
172
  if (this.currentSource) {
 
167
  }
168
  }
169
 
170
+ /** Check if currently playing audio (within last 300ms). */
171
+ isActive(): boolean {
172
+ return this.currentSource !== null && performance.now() - this.lastActiveAt < 300
173
+ }
174
+
175
  /** Stop current audio playback immediately. */
176
  stopAudio() {
177
  if (this.currentSource) {
src/components/friend/frontend/mmd-parser.d.ts DELETED
@@ -1,39 +0,0 @@
1
- declare module 'mmd-parser' {
2
- export const CharsetEncoder: any
3
-
4
- export class Parser {
5
- parseVmd(buffer: ArrayBufferLike, leftToRight?: boolean): VmdFile
6
- }
7
-
8
- export interface VmdFile {
9
- metadata: {
10
- magic: string
11
- name: string
12
- motionCount: number
13
- morphCount: number
14
- cameraCount: number
15
- coordinateSystem: string
16
- }
17
- motions: {
18
- boneName: string
19
- frameNum: number
20
- position: number[]
21
- rotation: number[]
22
- interpolation: number[]
23
- }[]
24
- morphs: {
25
- morphName: string
26
- frameNum: number
27
- weight: number
28
- }[]
29
- cameras: {
30
- frameNum: number
31
- distance: number
32
- position: number[]
33
- rotation: number[]
34
- interpolation: number[]
35
- fov: number
36
- perspective: number
37
- }[]
38
- }
39
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
src/components/friend/frontend/motion-controller.ts CHANGED
@@ -1,8 +1,10 @@
1
  /**
2
- * MotionController — unified animation system supporting VRMA, VMD, and FBX.
3
  *
4
  * Uses a single persistent AnimationMixer with crossFade transitions to avoid
5
  * T-pose flickering between animations.
 
 
6
  */
7
 
8
  import * as THREE from 'three'
@@ -10,14 +12,11 @@ import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js'
10
  import { VRMAnimationLoaderPlugin, createVRMAnimationClip } from '@pixiv/three-vrm-animation'
11
  import type { VRMAnimation } from '@pixiv/three-vrm-animation'
12
  import type { VRM } from '@pixiv/three-vrm'
13
- import { parseVMDAnimation, bindVMDToVRM } from './vmd-loader'
14
- import type { VMDAnimationData } from './vmd-loader'
15
  import { loadMixamoAnimation } from './mixamo-loader'
16
- import { VRMIKHandler } from './vrm-ik-handler'
17
 
18
  // ── Motion file types ───────────────────────────────────────────────────────
19
 
20
- export type MotionFileType = 'vrma' | 'vmd' | 'fbx'
21
 
22
  // ── Motion presets ──────────────────────────────────────────────────────────
23
 
@@ -29,13 +28,11 @@ export interface MotionPreset {
29
 
30
  // Actions: short one-shot gestures triggered by emotions / interactions
31
  export const actionPresets: Record<string, MotionPreset> = {
32
- // Local VRMA actions (bundled)
33
  akimbo: { label: '叉腰', type: 'vrma', url: '/friend/akimbo.vrma' },
34
  playFingers: { label: '搓手', type: 'vrma', url: '/friend/playFingers.vrma' },
35
  scratchHead: { label: '挠头', type: 'vrma', url: '/friend/scratchHead.vrma' },
36
  stretch: { label: '伸展', type: 'vrma', url: '/friend/stretch.vrma' },
37
 
38
- // Local FBX actions (bundled)
39
  happy: { label: '开心', type: 'fbx', url: '/friend/happy.fbx' },
40
  angry: { label: '生气', type: 'fbx', url: '/friend/angry.fbx' },
41
  greeting: { label: '招呼', type: 'fbx', url: '/friend/greeting.fbx' },
@@ -46,16 +43,6 @@ export const actionPresets: Record<string, MotionPreset> = {
46
  angryPump: { label: '暴怒', type: 'fbx', url: '/friend/angryPump.fbx' },
47
  }
48
 
49
- // Dances: looping full-body animations with optional BGM
50
- export interface DancePreset extends MotionPreset {
51
- bgm?: string
52
- }
53
-
54
- export const dancePresets: Record<string, DancePreset> = {
55
- jile: { label: '极乐净土', type: 'vmd', url: '/friend/jile.vmd', bgm: '/friend/jile.mp3' },
56
- love: { label: '恋爱循环', type: 'vmd', url: '/friend/love.vmd', bgm: '/friend/love.mp3' },
57
- }
58
-
59
  // ── Utility: re-anchor root position ────────────────────────────────────────
60
 
61
  function reAnchorRootPositionTrack(clip: THREE.AnimationClip, vrm: VRM) {
@@ -103,42 +90,44 @@ export class MotionController {
103
  private idleAction: THREE.AnimationAction | null = null
104
  private currentAction: THREE.AnimationAction | null = null
105
  private clipCache = new Map<string, THREE.AnimationClip>()
106
- private vmdDataCache = new Map<string, VMDAnimationData>()
107
  private gltfLoader: GLTFLoader
108
- private ikHandler: VRMIKHandler
109
- private _isDancing = false
110
  private _actionPlaying = false
111
- private _ikActive = false
112
  private holdTimer: ReturnType<typeof setTimeout> | null = null
113
  private _actionSafetyTimer: ReturnType<typeof setTimeout> | null = null
114
- private bgmAudio: HTMLAudioElement | null = null
115
-
116
- // Callbacks for external coordination (camera switching etc.)
117
- onDanceStart?: () => void
118
- onDanceStop?: () => void
119
 
120
  constructor(vrm: VRM) {
121
  this.vrm = vrm
122
  this.mixer = new THREE.AnimationMixer(vrm.scene)
123
  this.gltfLoader = new GLTFLoader()
124
  this.gltfLoader.register((parser) => new VRMAnimationLoaderPlugin(parser))
125
- this.ikHandler = VRMIKHandler.get(vrm)
126
  }
127
 
128
- private _volume = 0.5
129
-
130
- get isDancing() { return this._isDancing }
131
  get actionPlaying() { return this._actionPlaying }
132
 
133
- /** Set BGM volume (0–1). Also applies to currently playing BGM. */
134
- setVolume(v: number) {
135
- this._volume = v
136
- if (this.bgmAudio) this.bgmAudio.volume = v
 
137
  }
138
 
139
- update(delta: number) {
140
- if (this.mixer) this.mixer.update(delta)
141
- if (this._ikActive) this.ikHandler.update()
 
 
 
 
 
 
 
 
 
 
 
142
  }
143
 
144
  // ── CrossFade helper ─────────────────────────────────────────────────────
@@ -177,25 +166,17 @@ export class MotionController {
177
  }
178
 
179
  // ── Reset to idle (public) ──────────────────────────────────────────────
180
- // Full reset: stop BGM + stop dance + restart idle on a fresh mixer.
181
 
182
  resetToIdle() {
183
- this.stopBgm()
184
  this.clearTimers()
185
- this.disableIK()
186
-
187
- const wasDancing = this._isDancing
188
- this._isDancing = false
189
  this._actionPlaying = false
190
- this._actionGeneration++ // invalidate any in-flight playAction
191
 
192
  // CrossFade back to idle
193
  if (this.mixer && this.idleClip) {
194
  this.idleAction = this.mixer.clipAction(this.idleClip)
195
  this.crossFadeTo(this.idleAction)
196
  }
197
-
198
- if (wasDancing) this.onDanceStop?.()
199
  }
200
 
201
  // ── Play a one-shot action ──────────────────────────────────────────────
@@ -203,15 +184,24 @@ export class MotionController {
203
  private _actionGeneration = 0
204
 
205
  async playAction(name: string, hold = false) {
206
- if (this._actionPlaying) return
207
- if (this._isDancing) return
208
-
209
  const preset = actionPresets[name]
210
  if (!preset) { console.warn('[Motion] unknown action:', name); return }
211
 
 
 
 
 
 
 
 
 
 
 
212
  // Set lock BEFORE async load to prevent concurrent playAction calls
213
  this._actionPlaying = true
214
  const gen = ++this._actionGeneration
 
 
215
 
216
  const clip = await this.loadClip(preset)
217
 
@@ -224,108 +214,51 @@ export class MotionController {
224
 
225
  const action = this.mixer.clipAction(clip)
226
  action.setLoop(THREE.LoopOnce, 1)
227
- action.clampWhenFinished = true // always clamp to avoid T-pose on finish
228
  this.crossFadeTo(action)
229
 
230
- let settled = false
231
- const settle = () => {
232
- if (settled) return
233
- settled = true
234
- this.mixer?.removeEventListener('finished', onFinished)
235
- // Stale settle: another action or resetToIdle already took over
236
- if (gen !== this._actionGeneration) return
237
- this.clearTimers()
238
- if (hold) {
239
- this.holdTimer = setTimeout(() => {
240
- if (gen !== this._actionGeneration) return
241
- this._actionPlaying = false
242
- this.disableIK()
243
- this.startIdle()
244
- }, 10000)
245
- } else {
246
- this._actionPlaying = false
247
- this.disableIK()
248
- this.startIdle()
249
  }
250
- }
251
- const onFinished = () => settle()
252
- this.mixer?.addEventListener('finished', onFinished)
253
-
254
- // Safety timeout: guarantee _actionPlaying resets even if 'finished' never fires
255
- const duration = clip.duration > 0 ? clip.duration : 3
256
- this._actionSafetyTimer = setTimeout(() => settle(), (duration + 1) * 1000)
257
  }
258
 
259
- // ── Dance (looping VMD/FBX/VRMA) ───────────────────────────────────────
260
-
261
- async playDance(nameOrPreset: string | DancePreset) {
262
- if (this._isDancing) return
263
- this._isDancing = true
 
264
 
265
- try {
266
- // Accept preset name, URL, or full DancePreset object
267
- const preset: DancePreset | undefined =
268
- typeof nameOrPreset === 'object' ? nameOrPreset : dancePresets[nameOrPreset]
269
- const clip = preset
270
- ? await this.loadClip(preset)
271
- : await this.loadClipByUrl(nameOrPreset as string)
272
-
273
- if (!clip) {
274
- this._isDancing = false
275
- return
276
- }
277
 
278
- this.clearTimers()
 
 
279
  this._actionPlaying = false
280
-
281
- this.onDanceStart?.()
282
-
283
- // Play BGM if preset has one
284
- this.stopBgmImmediate()
285
- if (preset?.bgm) {
286
- this.bgmAudio = new Audio(preset.bgm)
287
- this.bgmAudio.loop = true
288
- this.bgmAudio.volume = this._volume
289
- this.bgmAudio.play().catch(() => {})
290
- }
291
-
292
- if (!this.mixer) { this._isDancing = false; return }
293
- const danceAction = this.mixer.clipAction(clip)
294
- danceAction.setLoop(THREE.LoopRepeat, Infinity)
295
- this.crossFadeTo(danceAction)
296
- } catch (err) {
297
- console.error('Failed to start dance:', err)
298
- this._isDancing = false
299
  }
300
- }
301
-
302
- /** Stop BGM with fade-out. Safe to call anytime. */
303
- private stopBgm() {
304
- if (!this.bgmAudio) return
305
- const audio = this.bgmAudio
306
- this.bgmAudio = null
307
- const fadeInterval = setInterval(() => {
308
- audio.volume = Math.max(0, audio.volume - 0.1)
309
- if (audio.volume <= 0) {
310
- clearInterval(fadeInterval)
311
- audio.pause()
312
- }
313
- }, 50)
314
- }
315
 
316
- /** Stop BGM instantly without fade. Used to prevent duplicate playback. */
317
- private stopBgmImmediate() {
318
- if (!this.bgmAudio) return
319
- this.bgmAudio.pause()
320
- this.bgmAudio = null
 
 
 
 
 
321
  }
322
 
323
  /** Cleanup when controller is being destroyed (model reload etc.) */
324
  dispose() {
325
- this.stopBgm()
326
  this.clearTimers()
327
- this.disableIK()
328
- this._isDancing = false
329
  this._actionPlaying = false
330
  this._actionGeneration++
331
  if (this.mixer) {
@@ -339,19 +272,7 @@ export class MotionController {
339
 
340
  // ── Internal ────────────────────────────────────────────────────────────
341
 
342
- private disableIK() {
343
- if (this._ikActive) {
344
- this._ikActive = false
345
- this.ikHandler.disableAll()
346
- }
347
- }
348
-
349
  private async loadClip(preset: MotionPreset): Promise<THREE.AnimationClip | null> {
350
- // For VMD, always rebuild clip with IK binding (IK targets are per-play)
351
- if (preset.type === 'vmd') {
352
- return this.loadVMDWithIK(preset.url)
353
- }
354
-
355
  const cached = this.clipCache.get(preset.url)
356
  if (cached) return cached
357
 
@@ -379,32 +300,6 @@ export class MotionController {
379
  return clip
380
  }
381
 
382
- private async loadVMDWithIK(url: string): Promise<THREE.AnimationClip | null> {
383
- try {
384
- // Cache parsed VMD data (parsing is expensive), but rebuild clip each time
385
- // because IK binding creates unique target objects per play
386
- let data = this.vmdDataCache.get(url)
387
- if (!data) {
388
- data = await parseVMDAnimation(url, this.vrm)
389
- this.vmdDataCache.set(url, data)
390
- }
391
- const clip = bindVMDToVRM(data, this.vrm, this.ikHandler)
392
- this._ikActive = true
393
- return clip
394
- } catch (err) {
395
- console.error('Failed to load VMD:', url, err)
396
- return null
397
- }
398
- }
399
-
400
- private async loadClipByUrl(url: string): Promise<THREE.AnimationClip | null> {
401
- const ext = url.split('.').pop()?.toLowerCase()
402
- const type: MotionFileType =
403
- ext === 'vmd' ? 'vmd' :
404
- ext === 'fbx' ? 'fbx' : 'vrma'
405
- return this.loadClip({ label: url, type, url })
406
- }
407
-
408
  private async loadVRMA(url: string): Promise<THREE.AnimationClip | null> {
409
  try {
410
  const gltf = await this.gltfLoader.loadAsync(url)
 
1
  /**
2
+ * MotionController — unified animation system supporting VRMA and FBX actions.
3
  *
4
  * Uses a single persistent AnimationMixer with crossFade transitions to avoid
5
  * T-pose flickering between animations.
6
+ *
7
+ * Simplified: removed dance system, kept action presets for human-like behavior.
8
  */
9
 
10
  import * as THREE from 'three'
 
12
  import { VRMAnimationLoaderPlugin, createVRMAnimationClip } from '@pixiv/three-vrm-animation'
13
  import type { VRMAnimation } from '@pixiv/three-vrm-animation'
14
  import type { VRM } from '@pixiv/three-vrm'
 
 
15
  import { loadMixamoAnimation } from './mixamo-loader'
 
16
 
17
  // ── Motion file types ───────────────────────────────────────────────────────
18
 
19
+ export type MotionFileType = 'vrma' | 'fbx'
20
 
21
  // ── Motion presets ──────────────────────────────────────────────────────────
22
 
 
28
 
29
  // Actions: short one-shot gestures triggered by emotions / interactions
30
  export const actionPresets: Record<string, MotionPreset> = {
 
31
  akimbo: { label: '叉腰', type: 'vrma', url: '/friend/akimbo.vrma' },
32
  playFingers: { label: '搓手', type: 'vrma', url: '/friend/playFingers.vrma' },
33
  scratchHead: { label: '挠头', type: 'vrma', url: '/friend/scratchHead.vrma' },
34
  stretch: { label: '伸展', type: 'vrma', url: '/friend/stretch.vrma' },
35
 
 
36
  happy: { label: '开心', type: 'fbx', url: '/friend/happy.fbx' },
37
  angry: { label: '生气', type: 'fbx', url: '/friend/angry.fbx' },
38
  greeting: { label: '招呼', type: 'fbx', url: '/friend/greeting.fbx' },
 
43
  angryPump: { label: '暴怒', type: 'fbx', url: '/friend/angryPump.fbx' },
44
  }
45
 
 
 
 
 
 
 
 
 
 
 
46
  // ── Utility: re-anchor root position ────────────────────────────────────────
47
 
48
  function reAnchorRootPositionTrack(clip: THREE.AnimationClip, vrm: VRM) {
 
90
  private idleAction: THREE.AnimationAction | null = null
91
  private currentAction: THREE.AnimationAction | null = null
92
  private clipCache = new Map<string, THREE.AnimationClip>()
 
93
  private gltfLoader: GLTFLoader
 
 
94
  private _actionPlaying = false
 
95
  private holdTimer: ReturnType<typeof setTimeout> | null = null
96
  private _actionSafetyTimer: ReturnType<typeof setTimeout> | null = null
97
+ private actionQueue: Array<{ name: string; hold: boolean }> = []
98
+ private _settleGen = 0 // generation at which the current settle is valid
99
+ private _settleHold = false // hold flag for current settle
 
 
100
 
101
  constructor(vrm: VRM) {
102
  this.vrm = vrm
103
  this.mixer = new THREE.AnimationMixer(vrm.scene)
104
  this.gltfLoader = new GLTFLoader()
105
  this.gltfLoader.register((parser) => new VRMAnimationLoaderPlugin(parser))
 
106
  }
107
 
 
 
 
108
  get actionPlaying() { return this._actionPlaying }
109
 
110
+ update(delta: number) {
111
+ if (this.mixer) {
112
+ this.mixer.update(delta)
113
+ this.checkActionCompletion()
114
+ }
115
  }
116
 
117
+ /**
118
+ * Frame-accurate action completion detection.
119
+ * THREE.AnimationMixer 'finished' event only fires when ALL actions finish,
120
+ * which never happens with a looping idle. So we check per-frame instead.
121
+ */
122
+ private checkActionCompletion() {
123
+ if (!this._actionPlaying || !this.currentAction) return
124
+ const clip = this.currentAction.getClip()
125
+ if (!clip || clip.duration <= 0) return
126
+
127
+ // Allow a 1-frame epsilon (≈16ms at 60fps) to avoid precision issues
128
+ if (this.currentAction.time >= clip.duration - 0.02) {
129
+ this.finishCurrentAction()
130
+ }
131
  }
132
 
133
  // ── CrossFade helper ─────────────────────────────────────────────────────
 
166
  }
167
 
168
  // ── Reset to idle (public) ──────────────────────────────────────────────
 
169
 
170
  resetToIdle() {
 
171
  this.clearTimers()
 
 
 
 
172
  this._actionPlaying = false
173
+ this._actionGeneration++
174
 
175
  // CrossFade back to idle
176
  if (this.mixer && this.idleClip) {
177
  this.idleAction = this.mixer.clipAction(this.idleClip)
178
  this.crossFadeTo(this.idleAction)
179
  }
 
 
180
  }
181
 
182
  // ── Play a one-shot action ──────────────────────────────────────────────
 
184
  private _actionGeneration = 0
185
 
186
  async playAction(name: string, hold = false) {
 
 
 
187
  const preset = actionPresets[name]
188
  if (!preset) { console.warn('[Motion] unknown action:', name); return }
189
 
190
+ // If already playing an action, queue instead of dropping
191
+ if (this._actionPlaying) {
192
+ // Avoid duplicate consecutive queued items
193
+ const last = this.actionQueue[this.actionQueue.length - 1]
194
+ if (!last || last.name !== name) {
195
+ this.actionQueue.push({ name, hold })
196
+ }
197
+ return
198
+ }
199
+
200
  // Set lock BEFORE async load to prevent concurrent playAction calls
201
  this._actionPlaying = true
202
  const gen = ++this._actionGeneration
203
+ this._settleGen = gen
204
+ this._settleHold = hold
205
 
206
  const clip = await this.loadClip(preset)
207
 
 
214
 
215
  const action = this.mixer.clipAction(clip)
216
  action.setLoop(THREE.LoopOnce, 1)
217
+ action.clampWhenFinished = true
218
  this.crossFadeTo(action)
219
 
220
+ // Frame-accurate completion is handled in checkActionCompletion() via update()
221
+ // Safety timer: generous fallback (should never fire if checkActionCompletion works)
222
+ const safeDuration = Math.max(clip.duration, 3) + 5
223
+ this._actionSafetyTimer = setTimeout(() => {
224
+ if (gen === this._actionGeneration && this._actionPlaying) {
225
+ this.finishCurrentAction()
 
 
 
 
 
 
 
 
 
 
 
 
 
226
  }
227
+ }, safeDuration * 1000)
 
 
 
 
 
 
228
  }
229
 
230
+ /** Called by checkActionCompletion() or safety timer when current action ends. */
231
+ private finishCurrentAction() {
232
+ if (!this._actionPlaying) return
233
+ this.clearTimers()
234
+ const gen = this._settleGen
235
+ const hold = this._settleHold
236
 
237
+ if (gen !== this._actionGeneration) return
 
 
 
 
 
 
 
 
 
 
 
238
 
239
+ // Process next queued action
240
+ const next = this.actionQueue.shift()
241
+ if (next) {
242
  this._actionPlaying = false
243
+ this.playAction(next.name, next.hold)
244
+ return
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
245
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
246
 
247
+ if (hold) {
248
+ this.holdTimer = setTimeout(() => {
249
+ if (gen !== this._actionGeneration) return
250
+ this._actionPlaying = false
251
+ this.startIdle()
252
+ }, 10000)
253
+ } else {
254
+ this._actionPlaying = false
255
+ this.startIdle()
256
+ }
257
  }
258
 
259
  /** Cleanup when controller is being destroyed (model reload etc.) */
260
  dispose() {
 
261
  this.clearTimers()
 
 
262
  this._actionPlaying = false
263
  this._actionGeneration++
264
  if (this.mixer) {
 
272
 
273
  // ── Internal ────────────────────────────────────────────────────────────
274
 
 
 
 
 
 
 
 
275
  private async loadClip(preset: MotionPreset): Promise<THREE.AnimationClip | null> {
 
 
 
 
 
276
  const cached = this.clipCache.get(preset.url)
277
  if (cached) return cached
278
 
 
300
  return clip
301
  }
302
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
303
  private async loadVRMA(url: string): Promise<THREE.AnimationClip | null> {
304
  try {
305
  const gltf = await this.gltfLoader.loadAsync(url)
src/components/friend/frontend/vmd-loader.ts DELETED
@@ -1,663 +0,0 @@
1
- /**
2
- * VMD animation loader for VRM models.
3
- * Ported from lobe-vidol's VMDAnimation module.
4
- *
5
- * Parses VMD binary files (via mmd-parser), converts bone/morph keyframes
6
- * to VRM-compatible AnimationClips with IK support.
7
- */
8
-
9
- import {
10
- VRMExpressionPresetName as BlendShapePresetName,
11
- VRMHumanBoneName as HumanoidBoneName,
12
- } from '@pixiv/three-vrm-core'
13
- import type { VRM, VRMHumanBoneName } from '@pixiv/three-vrm'
14
- import { CharsetEncoder, Parser } from 'mmd-parser'
15
- import type { VmdFile } from 'mmd-parser'
16
- import {
17
- AnimationClip,
18
- KeyframeTrack,
19
- MathUtils,
20
- NumberKeyframeTrack,
21
- Object3D,
22
- Quaternion,
23
- QuaternionKeyframeTrack,
24
- Vector3,
25
- VectorKeyframeTrack,
26
- } from 'three'
27
-
28
- import { VRMIKHandler } from './vrm-ik-handler'
29
-
30
- // ── Types ───────────────────────────────────────────────────────────────────
31
-
32
- export interface VMDAnimationData {
33
- duration: number
34
- timelines: Timeline[]
35
- }
36
-
37
- interface Timeline {
38
- name: VRMHumanBoneName | BlendShapePresetName
39
- type: 'position' | 'rotation' | 'morph'
40
- isIK?: boolean
41
- times: number[]
42
- values: number[]
43
- }
44
-
45
- interface VRMOffsets {
46
- hipsOffset?: number[]
47
- leftFootOffset?: number[]
48
- leftToeOffset?: number[]
49
- rightFootOffset?: number[]
50
- rightToeOffset?: number[]
51
- }
52
-
53
- interface Keyframe {
54
- boneName: string
55
- frameNum: number
56
- position: Vector3
57
- rotation: Quaternion
58
- }
59
-
60
- interface LerpKeyframe extends Keyframe {
61
- isNew?: boolean
62
- }
63
-
64
- interface IKOffsetInit {
65
- x: number
66
- y: number
67
- z: number
68
- s?: number
69
- sx?: number
70
- sy?: number
71
- sz?: number
72
- o?: number
73
- ox?: number
74
- oy?: number
75
- oz?: number
76
- dx?: boolean
77
- dy?: boolean
78
- dz?: boolean
79
- }
80
-
81
- // ── Hot-patch CharsetEncoder to forgive invalid charcodes ───────────────────
82
-
83
- {
84
- const { s2u } = CharsetEncoder.prototype
85
- CharsetEncoder.prototype.s2u = function (array: any) {
86
- try {
87
- return s2u.call(this, array)
88
- } catch (e) {
89
- if (typeof e === 'string') {
90
- console.warn(`Charset encoder: ${e}`)
91
- return ''
92
- }
93
- throw e
94
- }
95
- }
96
- }
97
-
98
- // ── Constants ───────────────────────────────────────────────────────────────
99
-
100
- function isTruely<T>(x: T): x is Exclude<T, false | '' | 0 | null | undefined> {
101
- return !!x
102
- }
103
-
104
- const tempV3 = new Vector3()
105
- const tempQ = new Quaternion()
106
-
107
- const VMDBoneNames = {
108
- Root: '全ての親',
109
- Center: 'センター',
110
- Hips: '下半身',
111
- Spine: '上半身',
112
- Chest: '上半身2',
113
- Neck: '首',
114
- Head: '頭',
115
- LeftEye: '左目',
116
- LeftShoulder: '左肩',
117
- LeftUpperArm: '左腕',
118
- LeftLowerArm: '左ひじ',
119
- LeftHand: '左手首',
120
- LeftThumbProximal: '左親指0',
121
- LeftThumbIntermediate: '左親指1',
122
- LeftThumbDistal: '左親指2',
123
- LeftIndexProximal: '左人指1',
124
- LeftIndexIntermediate: '左人指2',
125
- LeftIndexDistal: '左人指3',
126
- LeftMiddleProximal: '左中指1',
127
- LeftMiddleIntermediate: '左中指2',
128
- LeftMiddleDistal: '左中指3',
129
- LeftRingProximal: '左薬指1',
130
- LeftRingIntermediate: '左薬指2',
131
- LeftRingDistal: '左薬指3',
132
- LeftLittleProximal: '左小指1',
133
- LeftLittleIntermediate: '左小指2',
134
- LeftLittleDistal: '左小指3',
135
- LeftUpperLeg: '左足',
136
- LeftLowerLeg: '左ひざ',
137
- LeftFoot: '左足首',
138
- LeftFootIK: '左足IK',
139
- LeftToes: '左つま先',
140
- LeftToeIK: '左つま先IK',
141
- RightEye: '右目',
142
- RightShoulder: '右肩',
143
- RightUpperArm: '右腕',
144
- RightLowerArm: '右ひじ',
145
- RightHand: '右手首',
146
- RightThumbProximal: '右親指0',
147
- RightThumbIntermediate: '右親指1',
148
- RightThumbDistal: '右親指2',
149
- RightIndexProximal: '右人指1',
150
- RightIndexIntermediate: '右人指2',
151
- RightIndexDistal: '右人指3',
152
- RightMiddleProximal: '右中指1',
153
- RightMiddleIntermediate: '右中指2',
154
- RightMiddleDistal: '右中指3',
155
- RightRingProximal: '右薬指1',
156
- RightRingIntermediate: '右薬指2',
157
- RightRingDistal: '右薬指3',
158
- RightLittleProximal: '右小指1',
159
- RightLittleIntermediate: '右小指2',
160
- RightLittleDistal: '右小指3',
161
- RightUpperLeg: '右足',
162
- RightLowerLeg: '右ひざ',
163
- RightFoot: '右足首',
164
- RightFootIK: '右足IK',
165
- RightToes: '右つま先',
166
- RightToeIK: '右つま先IK',
167
- } as const
168
-
169
- const VMDMorphNames = {
170
- Blink: 'まばたき',
171
- BlinkR: 'ウィンク',
172
- BlinkL: 'ウィンク右',
173
- A: 'あ',
174
- I: 'い',
175
- U: 'う',
176
- E: 'え',
177
- O: 'お',
178
- } as const
179
-
180
- const VRM_VMD_BONE_MAP = new Map<string, HumanoidBoneName>([
181
- [VMDBoneNames.Hips, HumanoidBoneName.Hips],
182
- [VMDBoneNames.Spine, HumanoidBoneName.Spine],
183
- [VMDBoneNames.Chest, HumanoidBoneName.Chest],
184
- [VMDBoneNames.Neck, HumanoidBoneName.Neck],
185
- [VMDBoneNames.Head, HumanoidBoneName.Head],
186
- [VMDBoneNames.LeftEye, HumanoidBoneName.LeftEye],
187
- [VMDBoneNames.LeftShoulder, HumanoidBoneName.LeftShoulder],
188
- [VMDBoneNames.LeftUpperArm, HumanoidBoneName.LeftUpperArm],
189
- [VMDBoneNames.LeftLowerArm, HumanoidBoneName.LeftLowerArm],
190
- [VMDBoneNames.LeftHand, HumanoidBoneName.LeftHand],
191
- [VMDBoneNames.LeftThumbProximal, HumanoidBoneName.LeftThumbProximal],
192
- [VMDBoneNames.LeftThumbIntermediate, HumanoidBoneName.LeftThumbMetacarpal],
193
- [VMDBoneNames.LeftThumbDistal, HumanoidBoneName.LeftThumbDistal],
194
- [VMDBoneNames.LeftIndexProximal, HumanoidBoneName.LeftIndexProximal],
195
- [VMDBoneNames.LeftIndexIntermediate, HumanoidBoneName.LeftIndexIntermediate],
196
- [VMDBoneNames.LeftIndexDistal, HumanoidBoneName.LeftIndexDistal],
197
- [VMDBoneNames.LeftMiddleProximal, HumanoidBoneName.LeftMiddleProximal],
198
- [VMDBoneNames.LeftMiddleIntermediate, HumanoidBoneName.LeftMiddleIntermediate],
199
- [VMDBoneNames.LeftMiddleDistal, HumanoidBoneName.LeftMiddleDistal],
200
- [VMDBoneNames.LeftRingProximal, HumanoidBoneName.LeftRingProximal],
201
- [VMDBoneNames.LeftRingIntermediate, HumanoidBoneName.LeftRingIntermediate],
202
- [VMDBoneNames.LeftRingDistal, HumanoidBoneName.LeftRingDistal],
203
- [VMDBoneNames.LeftLittleProximal, HumanoidBoneName.LeftLittleProximal],
204
- [VMDBoneNames.LeftLittleIntermediate, HumanoidBoneName.LeftLittleIntermediate],
205
- [VMDBoneNames.LeftLittleDistal, HumanoidBoneName.LeftLittleDistal],
206
- [VMDBoneNames.LeftUpperLeg, HumanoidBoneName.LeftUpperLeg],
207
- [VMDBoneNames.LeftLowerLeg, HumanoidBoneName.LeftLowerLeg],
208
- [VMDBoneNames.LeftFoot, HumanoidBoneName.LeftFoot],
209
- [VMDBoneNames.LeftToes, HumanoidBoneName.LeftToes],
210
- [VMDBoneNames.RightEye, HumanoidBoneName.RightEye],
211
- [VMDBoneNames.RightShoulder, HumanoidBoneName.RightShoulder],
212
- [VMDBoneNames.RightUpperArm, HumanoidBoneName.RightUpperArm],
213
- [VMDBoneNames.RightLowerArm, HumanoidBoneName.RightLowerArm],
214
- [VMDBoneNames.RightHand, HumanoidBoneName.RightHand],
215
- [VMDBoneNames.RightThumbProximal, HumanoidBoneName.RightThumbProximal],
216
- [VMDBoneNames.RightThumbIntermediate, HumanoidBoneName.RightThumbMetacarpal],
217
- [VMDBoneNames.RightThumbDistal, HumanoidBoneName.RightThumbDistal],
218
- [VMDBoneNames.RightIndexProximal, HumanoidBoneName.RightIndexProximal],
219
- [VMDBoneNames.RightIndexIntermediate, HumanoidBoneName.RightIndexIntermediate],
220
- [VMDBoneNames.RightIndexDistal, HumanoidBoneName.RightIndexDistal],
221
- [VMDBoneNames.RightMiddleProximal, HumanoidBoneName.RightMiddleProximal],
222
- [VMDBoneNames.RightMiddleIntermediate, HumanoidBoneName.RightMiddleIntermediate],
223
- [VMDBoneNames.RightMiddleDistal, HumanoidBoneName.RightMiddleDistal],
224
- [VMDBoneNames.RightRingProximal, HumanoidBoneName.RightRingProximal],
225
- [VMDBoneNames.RightRingIntermediate, HumanoidBoneName.RightRingIntermediate],
226
- [VMDBoneNames.RightRingDistal, HumanoidBoneName.RightRingDistal],
227
- [VMDBoneNames.RightLittleProximal, HumanoidBoneName.RightLittleProximal],
228
- [VMDBoneNames.RightLittleIntermediate, HumanoidBoneName.RightLittleIntermediate],
229
- [VMDBoneNames.RightLittleDistal, HumanoidBoneName.RightLittleDistal],
230
- [VMDBoneNames.RightUpperLeg, HumanoidBoneName.RightUpperLeg],
231
- [VMDBoneNames.RightLowerLeg, HumanoidBoneName.RightLowerLeg],
232
- [VMDBoneNames.RightFoot, HumanoidBoneName.RightFoot],
233
- [VMDBoneNames.RightToes, HumanoidBoneName.RightToes],
234
- ])
235
-
236
- const VMD_VRM_IK_MAP = new Map<string, HumanoidBoneName>([
237
- [VMDBoneNames.LeftFootIK, HumanoidBoneName.LeftFoot],
238
- [VMDBoneNames.LeftToeIK, HumanoidBoneName.LeftToes],
239
- [VMDBoneNames.RightFootIK, HumanoidBoneName.RightFoot],
240
- [VMDBoneNames.RightToeIK, HumanoidBoneName.RightToes],
241
- ])
242
-
243
- const VMD_BONE_NAMES = new Set<string>(VRM_VMD_BONE_MAP.keys())
244
- Array.from(VMD_VRM_IK_MAP.keys()).forEach(VMD_BONE_NAMES.add, VMD_BONE_NAMES)
245
- VMD_BONE_NAMES.add(VMDBoneNames.Root)
246
- VMD_BONE_NAMES.add(VMDBoneNames.Center)
247
-
248
- const VMD_VRM_MORPH_MAP = new Map<string, BlendShapePresetName>([
249
- [VMDMorphNames.Blink, BlendShapePresetName.Blink],
250
- [VMDMorphNames.BlinkL, BlendShapePresetName.BlinkLeft],
251
- [VMDMorphNames.BlinkR, BlendShapePresetName.BlinkRight],
252
- [VMDMorphNames.A, BlendShapePresetName.Aa],
253
- [VMDMorphNames.I, BlendShapePresetName.Ih],
254
- [VMDMorphNames.U, BlendShapePresetName.Ou],
255
- [VMDMorphNames.E, BlendShapePresetName.Ee],
256
- [VMDMorphNames.O, BlendShapePresetName.Oh],
257
- ])
258
-
259
- const IK_OFFSET_INIT = new Map<string, IKOffsetInit>([
260
- [VMDBoneNames.Center, { x: 0, y: 1, z: 0, s: 10 }],
261
- [VMDBoneNames.LeftFootIK, { x: 1, y: 1, z: 0, s: 10, dx: true }],
262
- [VMDBoneNames.RightFootIK, { x: -1, y: 1, z: 0, s: 10, dx: true }],
263
- [VMDBoneNames.LeftToeIK, { x: 0, y: -1, z: -1, s: 10, oy: 2.5, dx: true, dz: true }],
264
- [VMDBoneNames.RightToeIK, { x: 0, y: -1, z: -1, s: 10, oy: 2.5, dx: true, dz: true }],
265
- ])
266
-
267
- const V3_ZERO = new Vector3()
268
- const Q_IDENTITY = new Quaternion()
269
- const Z_30_DEG_CW = new Quaternion().setFromAxisAngle(tempV3.set(0, 0, 1), 30 * MathUtils.DEG2RAD)
270
- const Z_30_DEG_CCW = Z_30_DEG_CW.clone().invert()
271
-
272
- // ── VMD Parsing ────────��────────────────────────────────────────────────────
273
-
274
- function convertVMD(buffer: ArrayBufferLike, vrmOffset?: VRMOffsets): VMDAnimationData {
275
- const vmd = new Parser().parseVmd(buffer)
276
- const morphs = convertMorphs(vmd)
277
- const motions = convertMotions(vmd, vrmOffset)
278
- return {
279
- duration: Math.max(morphs.duration, motions.duration),
280
- timelines: Array.prototype.concat(morphs.timelines, motions.timelines),
281
- }
282
- }
283
-
284
- function convertMorphs({ morphs }: VmdFile): VMDAnimationData {
285
- sortFrames(morphs)
286
- const timelines = new Map<string, Timeline>()
287
- for (const { morphName, weight, frameNum } of morphs) {
288
- const name = VMD_VRM_MORPH_MAP.get(morphName)
289
- if (!name) continue
290
- let timeline = timelines.get(name)
291
- if (!timeline)
292
- timelines.set(name, (timeline = { name, type: 'morph', times: [], values: [] }))
293
- const { times, values } = timeline
294
- const time = frameNum / 30
295
- const timeIndex = times.findIndex((t) => t === time)
296
- if (timeIndex < 0) {
297
- times.push(time)
298
- values.push(weight)
299
- } else {
300
- values[timeIndex] = Math.max(values[timeIndex], weight)
301
- }
302
- }
303
- return {
304
- timelines: Array.from(timelines.values()),
305
- duration: getLastFrameNum(morphs) / 30,
306
- }
307
- }
308
-
309
- function convertMotions({ motions }: VmdFile, vrmOffset?: VRMOffsets): VMDAnimationData {
310
- sortFrames(motions)
311
- const timelines: Timeline[] = []
312
- const boneTlMap = new Map<string, Keyframe[]>()
313
- for (const name of VMD_BONE_NAMES) boneTlMap.set(name, [])
314
- for (const { boneName, frameNum, position, rotation } of motions)
315
- boneTlMap.get(boneName)?.push({
316
- boneName,
317
- frameNum,
318
- position: new Vector3().fromArray(position),
319
- rotation: new Quaternion().fromArray(rotation),
320
- })
321
- fixPositions(boneTlMap, vrmOffset)
322
- for (const [boneName, timeline] of boneTlMap) {
323
- let name = VRM_VMD_BONE_MAP.get(boneName)
324
- let isIK = false
325
- if (!name) {
326
- isIK = VMD_VRM_IK_MAP.has(boneName)
327
- name = VMD_VRM_IK_MAP.get(boneName)
328
- }
329
- if (name) {
330
- const times: number[] = []
331
- const positions: number[] = []
332
- const rotations: number[] = []
333
- for (const f of timeline) {
334
- const i = times.push(f.frameNum / 30) - 1
335
- f.position.toArray(positions, i * 3)
336
- f.rotation.toArray(rotations, i * 4)
337
- }
338
- if (times.length) {
339
- timelines.push({ name, type: 'rotation', isIK, times, values: rotations })
340
- if (isIK || name === HumanoidBoneName.Hips)
341
- timelines.push({ name, type: 'position', isIK, times, values: positions })
342
- }
343
- }
344
- }
345
- return { timelines, duration: getLastFrameNum(motions) / 30 }
346
- }
347
-
348
- function fixPositions(tls: Map<string, Keyframe[]>, vrmOffset: VRMOffsets = {}) {
349
- const centerOffset = mergeTimelines(
350
- tls,
351
- VMDBoneNames.Center,
352
- offsetToTimeline(VMDBoneNames.Center, vrmOffset.hipsOffset),
353
- )
354
- const hipsTl = mergeTimelines(tls, VMDBoneNames.Root, centerOffset, VMDBoneNames.Hips)
355
- tls.set(
356
- VMDBoneNames.Spine,
357
- localizeTimeline(
358
- hipsTl,
359
- mergeTimelines(tls, VMDBoneNames.Root, centerOffset, VMDBoneNames.Hips),
360
- ),
361
- )
362
- tls.set(VMDBoneNames.Hips, hipsTl)
363
- const leftFootOffset = offsetToTimeline(VMDBoneNames.LeftFootIK, vrmOffset.leftFootOffset)
364
- const rightFootOffset = offsetToTimeline(VMDBoneNames.RightFootIK, vrmOffset.rightFootOffset)
365
- if (tls.has(VMDBoneNames.LeftToeIK))
366
- tls.set(
367
- VMDBoneNames.LeftToeIK,
368
- mergeTimelines(
369
- tls,
370
- VMDBoneNames.Root,
371
- leftFootOffset,
372
- VMDBoneNames.LeftFootIK,
373
- offsetToTimeline(VMDBoneNames.RightToeIK, vrmOffset.leftToeOffset),
374
- VMDBoneNames.LeftToeIK,
375
- ),
376
- )
377
- if (tls.has(VMDBoneNames.RightToeIK))
378
- tls.set(
379
- VMDBoneNames.RightToeIK,
380
- mergeTimelines(
381
- tls,
382
- VMDBoneNames.Root,
383
- rightFootOffset,
384
- VMDBoneNames.RightFootIK,
385
- offsetToTimeline(VMDBoneNames.RightToeIK, vrmOffset.rightToeOffset),
386
- VMDBoneNames.RightToeIK,
387
- ),
388
- )
389
- if (tls.has(VMDBoneNames.LeftFootIK))
390
- tls.set(
391
- VMDBoneNames.LeftFootIK,
392
- mergeTimelines(tls, VMDBoneNames.Root, leftFootOffset, VMDBoneNames.LeftFootIK),
393
- )
394
- if (tls.has(VMDBoneNames.RightFootIK))
395
- tls.set(
396
- VMDBoneNames.RightFootIK,
397
- mergeTimelines(tls, VMDBoneNames.Root, rightFootOffset, VMDBoneNames.RightFootIK),
398
- )
399
- tls.delete(VMDBoneNames.Center)
400
- tls.delete(VMDBoneNames.Root)
401
- for (const tl of tls.values())
402
- for (const f of tl) {
403
- f.position.x *= -1
404
- f.rotation.x *= -1
405
- f.rotation.w *= -1
406
- switch (f.boneName) {
407
- case VMDBoneNames.LeftUpperArm:
408
- f.rotation.multiply(Z_30_DEG_CW)
409
- break
410
- case VMDBoneNames.RightUpperArm:
411
- f.rotation.multiply(Z_30_DEG_CCW)
412
- break
413
- case VMDBoneNames.LeftLowerArm:
414
- case VMDBoneNames.LeftHand:
415
- case VMDBoneNames.LeftThumbProximal:
416
- case VMDBoneNames.LeftThumbIntermediate:
417
- case VMDBoneNames.LeftThumbDistal:
418
- case VMDBoneNames.LeftIndexProximal:
419
- case VMDBoneNames.LeftIndexIntermediate:
420
- case VMDBoneNames.LeftIndexDistal:
421
- case VMDBoneNames.LeftMiddleProximal:
422
- case VMDBoneNames.LeftMiddleIntermediate:
423
- case VMDBoneNames.LeftMiddleDistal:
424
- case VMDBoneNames.LeftRingProximal:
425
- case VMDBoneNames.LeftRingIntermediate:
426
- case VMDBoneNames.LeftRingDistal:
427
- case VMDBoneNames.LeftLittleProximal:
428
- case VMDBoneNames.LeftLittleIntermediate:
429
- case VMDBoneNames.LeftLittleDistal:
430
- f.rotation.premultiply(Z_30_DEG_CCW).multiply(Z_30_DEG_CW)
431
- break
432
- case VMDBoneNames.RightLowerArm:
433
- case VMDBoneNames.RightHand:
434
- case VMDBoneNames.RightThumbProximal:
435
- case VMDBoneNames.RightThumbIntermediate:
436
- case VMDBoneNames.RightThumbDistal:
437
- case VMDBoneNames.RightIndexProximal:
438
- case VMDBoneNames.RightIndexIntermediate:
439
- case VMDBoneNames.RightIndexDistal:
440
- case VMDBoneNames.RightMiddleProximal:
441
- case VMDBoneNames.RightMiddleIntermediate:
442
- case VMDBoneNames.RightMiddleDistal:
443
- case VMDBoneNames.RightRingProximal:
444
- case VMDBoneNames.RightRingIntermediate:
445
- case VMDBoneNames.RightRingDistal:
446
- case VMDBoneNames.RightLittleProximal:
447
- case VMDBoneNames.RightLittleIntermediate:
448
- case VMDBoneNames.RightLittleDistal:
449
- f.rotation.premultiply(Z_30_DEG_CW).multiply(Z_30_DEG_CCW)
450
- break
451
- }
452
- f.position.multiplyScalar(0.1)
453
- }
454
- }
455
-
456
- function offsetToTimeline(boneName: string, rawPos: number[] | undefined): Keyframe[] {
457
- const init = IK_OFFSET_INIT.get(boneName)!
458
- return [
459
- {
460
- boneName: `${boneName}Offset`,
461
- frameNum: 0,
462
- position: rawPos
463
- ? new Vector3(
464
- init.dx || isNaN(rawPos[0])
465
- ? init.x
466
- : rawPos[0] * (init.sx ?? 1) * (init.s ?? 1) + (init.ox ?? 0) + (init.o ?? 0),
467
- init.dy || isNaN(rawPos[1])
468
- ? init.y
469
- : rawPos[1] * (init.sy ?? 1) * (init.s ?? 1) + (init.oy ?? 0) + (init.o ?? 0),
470
- init.dz || isNaN(rawPos[2])
471
- ? init.z
472
- : rawPos[2] * (init.sz ?? 1) * (init.s ?? 1) + (init.oz ?? 0) + (init.o ?? 0),
473
- )
474
- : new Vector3(init.x, init.y, init.z),
475
- rotation: Q_IDENTITY,
476
- },
477
- ]
478
- }
479
-
480
- function mergeTimelines(tlsMap: Map<string, Keyframe[]>, ...tlsKey: (Keyframe[] | string)[]) {
481
- const tls = tlsKey.map(resolveTimeline, tlsMap).filter(isTruely)
482
- let boneName: string
483
- const last = tlsKey[tlsKey.length - 1]
484
- if (typeof last === 'string') boneName = last
485
- else boneName = tls[tls.length - 1][0]?.boneName ?? ''
486
- const results: Keyframe[] = []
487
- for (const tl of tls)
488
- for (const f of tl) {
489
- const { frameNum } = f
490
- if (frameNum < results.length && results[frameNum] != null) continue
491
- const position = new Vector3()
492
- const rotation = new Quaternion()
493
- for (const otl of tls) {
494
- if (!otl.length) continue
495
- const f2 = otl[0].boneName === f.boneName ? f : lerpKeyframe(otl, frameNum)
496
- position.add(tempV3.copy(f2.position).applyQuaternion(rotation))
497
- rotation.multiply(f2.rotation)
498
- }
499
- results[frameNum] = { boneName, frameNum, position, rotation }
500
- }
501
- return results.filter(isTruely)
502
- }
503
-
504
- function localizeTimeline(parent: Keyframe[], child: Keyframe[]): Keyframe[]
505
- function localizeTimeline(...tls: [Keyframe[], Keyframe[]]) {
506
- const { boneName } = tls[1][0]
507
- const results: Keyframe[] = []
508
- let isChild = false
509
- for (const tl of tls) {
510
- for (const f of tl) {
511
- const { frameNum } = f
512
- if (frameNum < results.length && results[frameNum] != null) continue
513
- const fp: LerpKeyframe = isChild ? lerpKeyframe(tls[0], frameNum) : f
514
- const fc: LerpKeyframe = isChild ? f : lerpKeyframe(tls[1], frameNum)
515
- results[frameNum] = {
516
- boneName,
517
- frameNum,
518
- position: (fc.isNew ? fc.position : fc.position.clone()).sub(fp.position),
519
- rotation: (fc.isNew ? fc.rotation : fc.rotation.clone()).multiply(
520
- tempQ.copy(fp.rotation).invert(),
521
- ),
522
- }
523
- }
524
- isChild = true
525
- }
526
- return results.filter(isTruely)
527
- }
528
-
529
- function lerpKeyframe(tl: Keyframe[], frameNum: number): LerpKeyframe {
530
- if (!tl)
531
- return { boneName: '', frameNum, position: V3_ZERO, rotation: Q_IDENTITY }
532
- const nextIndex = tl.findIndex((keyframe) => frameNum < keyframe.frameNum)
533
- switch (nextIndex) {
534
- case 0:
535
- return tl[0]
536
- case -1:
537
- return tl[tl.length - 1]
538
- case frameNum:
539
- return tl[frameNum]
540
- }
541
- const prevFrame = tl[nextIndex - 1]
542
- const nextFrame = tl[nextIndex]
543
- const prevFrameNum = prevFrame.frameNum
544
- const nextFrameNum = nextFrame.frameNum
545
- const v = (frameNum - prevFrameNum) / (nextFrameNum - prevFrameNum)
546
- return {
547
- boneName: tl[0].boneName,
548
- frameNum,
549
- position: prevFrame.position.clone().lerp(nextFrame.position, v),
550
- rotation: prevFrame.rotation.clone().slerp(nextFrame.rotation, v),
551
- isNew: true,
552
- }
553
- }
554
-
555
- function resolveTimeline(this: Map<string, Keyframe[]>, key: Keyframe[] | string) {
556
- return Array.isArray(key) ? key : this.get(key)
557
- }
558
-
559
- function sortFrames<T extends { frameNum: number }>(f: ArrayLike<T> | Iterable<T>) {
560
- return (Array.isArray(f) ? (f as T[]) : Array.from(f)).sort((a, b) => a.frameNum - b.frameNum)
561
- }
562
-
563
- function getLastFrameNum(f: { frameNum: number }[]) {
564
- return f.length ? f[f.length - 1].frameNum : 0
565
- }
566
-
567
- // ── VRM Offset Calculation ──────────────────────────────────────────────────
568
-
569
- function calculatePosition(from?: Object3D | null, to?: Object3D | null) {
570
- if (!from || !to) return
571
- let current: Object3D | null = to
572
- const chain: Object3D[] = [to]
573
- while (current.parent && current !== from) {
574
- chain.push(current.parent)
575
- current = current.parent
576
- }
577
- if (current === null) return
578
- chain.reverse()
579
- const position = tempV3.set(0, 0, 0)
580
- for (const node of chain) position.add(node.position)
581
- return position.toArray()
582
- }
583
-
584
- function toOffset(vrm: VRM): VRMOffsets {
585
- const { humanoid } = vrm
586
- if (!humanoid) throw new Error('VRM does not have humanoid')
587
- const currentPose = humanoid.getNormalizedPose()
588
- humanoid.resetNormalizedPose()
589
- const hips = humanoid.getNormalizedBoneNode(HumanoidBoneName.Hips)
590
- const leftFoot = humanoid.getNormalizedBoneNode(HumanoidBoneName.LeftFoot)
591
- const leftToe = humanoid.getNormalizedBoneNode(HumanoidBoneName.LeftToes)
592
- const rightFoot = humanoid.getNormalizedBoneNode(HumanoidBoneName.RightFoot)
593
- const rightToe = humanoid.getNormalizedBoneNode(HumanoidBoneName.RightToes)
594
- humanoid.setNormalizedPose(currentPose)
595
- return {
596
- hipsOffset: calculatePosition(hips, hips),
597
- leftFootOffset: calculatePosition(hips, leftFoot),
598
- leftToeOffset: calculatePosition(leftFoot, leftToe),
599
- rightFootOffset: calculatePosition(hips, rightFoot),
600
- rightToeOffset: calculatePosition(rightFoot, rightToe),
601
- }
602
- }
603
-
604
- // ── VRM Binding ─────────────────────────────────────────────────────────────
605
-
606
- function bindToVRM(data: VMDAnimationData, vrm: VRM): AnimationClip {
607
- const tracks: KeyframeTrack[] = []
608
- for (const { type, name, isIK, times, values } of data.timelines) {
609
- let srcName: string
610
- switch (type) {
611
- case 'morph': {
612
- const track = vrm.expressionManager?.getExpressionTrackName(name as BlendShapePresetName)
613
- if (!track) continue
614
- srcName = track
615
- break
616
- }
617
- case 'position':
618
- case 'rotation': {
619
- if (isIK) {
620
- const handler = VRMIKHandler.get(vrm)
621
- const target = handler.getAndEnableIK(name as VRMHumanBoneName)
622
- if (!target) continue
623
- srcName = target.name
624
- } else {
625
- const bone = vrm.humanoid?.getNormalizedBone(name as VRMHumanBoneName)
626
- if (!bone) continue
627
- srcName = bone.node.name
628
- }
629
- break
630
- }
631
- default:
632
- continue
633
- }
634
- switch (type) {
635
- case 'morph':
636
- tracks.push(new NumberKeyframeTrack(srcName, times, values))
637
- break
638
- case 'position':
639
- tracks.push(new VectorKeyframeTrack(`${srcName}.position`, times, values))
640
- break
641
- case 'rotation':
642
- tracks.push(new QuaternionKeyframeTrack(`${srcName}.quaternion`, times, values))
643
- break
644
- }
645
- }
646
- return new AnimationClip(`clip${Date.now()}`, data.duration, tracks)
647
- }
648
-
649
- // ── Public API (matches motion-controller imports) ──────────────────────────
650
-
651
- export async function parseVMDAnimation(url: string, vrm: VRM): Promise<VMDAnimationData> {
652
- const res = await fetch(url)
653
- const buffer = await res.arrayBuffer()
654
- return convertVMD(buffer, toOffset(vrm))
655
- }
656
-
657
- export function bindVMDToVRM(
658
- data: VMDAnimationData,
659
- vrm: VRM,
660
- _ikHandler: VRMIKHandler,
661
- ): AnimationClip {
662
- return bindToVRM(data, vrm)
663
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
src/components/friend/frontend/vrm-ik-handler.ts DELETED
@@ -1,236 +0,0 @@
1
- import { VRM, VRMHumanBoneName } from '@pixiv/three-vrm'
2
- import { Bone, MathUtils, Object3D, Quaternion, Vector3 } from 'three'
3
-
4
- import { clampVector3ByRadian } from './three-helpers'
5
-
6
- const BoneNames = VRMHumanBoneName
7
- const boneNameOrder: VRMHumanBoneName[] = [
8
- BoneNames.Chest,
9
- BoneNames.Head,
10
- BoneNames.Hips,
11
- BoneNames.Jaw,
12
- BoneNames.LeftEye,
13
- BoneNames.LeftFoot,
14
- BoneNames.LeftHand,
15
- BoneNames.LeftIndexDistal,
16
- BoneNames.LeftIndexIntermediate,
17
- BoneNames.LeftIndexProximal,
18
- BoneNames.LeftLittleDistal,
19
- BoneNames.LeftLittleIntermediate,
20
- BoneNames.LeftLittleProximal,
21
- BoneNames.LeftLowerArm,
22
- BoneNames.LeftLowerLeg,
23
- BoneNames.LeftMiddleDistal,
24
- BoneNames.LeftMiddleIntermediate,
25
- BoneNames.LeftMiddleProximal,
26
- BoneNames.LeftRingDistal,
27
- BoneNames.LeftRingIntermediate,
28
- BoneNames.LeftRingProximal,
29
- BoneNames.LeftShoulder,
30
- BoneNames.LeftThumbDistal,
31
- BoneNames.LeftThumbMetacarpal,
32
- BoneNames.LeftThumbProximal,
33
- BoneNames.LeftToes,
34
- BoneNames.LeftUpperArm,
35
- BoneNames.LeftUpperLeg,
36
- BoneNames.Neck,
37
- BoneNames.RightEye,
38
- BoneNames.RightFoot,
39
- BoneNames.RightHand,
40
- BoneNames.RightIndexDistal,
41
- BoneNames.RightIndexIntermediate,
42
- BoneNames.RightIndexProximal,
43
- BoneNames.RightLittleDistal,
44
- BoneNames.RightLittleIntermediate,
45
- BoneNames.RightLittleProximal,
46
- BoneNames.RightLowerArm,
47
- BoneNames.RightLowerLeg,
48
- BoneNames.RightMiddleDistal,
49
- BoneNames.RightMiddleIntermediate,
50
- BoneNames.RightMiddleProximal,
51
- BoneNames.RightRingDistal,
52
- BoneNames.RightRingIntermediate,
53
- BoneNames.RightRingProximal,
54
- BoneNames.RightShoulder,
55
- BoneNames.RightThumbDistal,
56
- BoneNames.RightThumbMetacarpal,
57
- BoneNames.RightThumbProximal,
58
- BoneNames.RightToes,
59
- BoneNames.RightUpperArm,
60
- BoneNames.RightUpperLeg,
61
- BoneNames.Spine,
62
- BoneNames.UpperChest,
63
- ]
64
- const boneMap = new Map<VRMHumanBoneName, number>(boneNameOrder.map((b, i) => [b, i]))
65
-
66
- interface IKS {
67
- effector: number
68
- target: number
69
- iteration: number
70
- links: {
71
- enabled: boolean
72
- index: number
73
- rotationMin?: Vector3
74
- rotationMax?: Vector3
75
- }[]
76
- minAngle?: number
77
- maxAngle?: number
78
- }
79
-
80
- const quaternion = new Quaternion()
81
- const targetPos = new Vector3()
82
- const targetVec = new Vector3()
83
- const effectorPos = new Vector3()
84
- const effectorVec = new Vector3()
85
- const linkPos = new Vector3()
86
- const linkScale = new Vector3()
87
- const axis = new Vector3()
88
-
89
- export class VRMIKHandler {
90
- private static cache = new WeakMap<VRM, VRMIKHandler>()
91
-
92
- static get(model: VRM) {
93
- let handler = this.cache.get(model)
94
- if (!handler) {
95
- handler = new VRMIKHandler(model)
96
- this.cache.set(model, handler)
97
- }
98
- return handler
99
- }
100
-
101
- private targets = new Map<number, Object3D>()
102
- private iks = new Map<number, IKS>()
103
- private bones: Bone[]
104
- private root: Object3D
105
-
106
- private constructor(public model: VRM) {
107
- const { humanoid } = this.model
108
- if (!humanoid) throw new Error('VRM does not contains humanoid')
109
- this.bones = boneNameOrder.map(humanoid.getRawBoneNode, humanoid) as Bone[]
110
- this.root = this.bones[boneMap.get(BoneNames.Hips)!]?.parent ?? this.model.scene
111
- const leftFootId = boneMap.get(BoneNames.LeftFoot)!
112
- this.iks.set(leftFootId, {
113
- effector: leftFootId,
114
- target: leftFootId,
115
- iteration: 40,
116
- maxAngle: 0.5,
117
- links: [
118
- {
119
- enabled: false,
120
- index: boneMap.get(BoneNames.LeftLowerLeg)!,
121
- rotationMin: new Vector3(-180, 0, 0).multiplyScalar(MathUtils.DEG2RAD),
122
- rotationMax: new Vector3(0, 0, 0),
123
- },
124
- {
125
- enabled: false,
126
- index: boneMap.get(BoneNames.LeftUpperLeg)!,
127
- },
128
- ],
129
- })
130
- const rightFootId = boneMap.get(BoneNames.RightFoot)!
131
- this.iks.set(rightFootId, {
132
- effector: rightFootId,
133
- target: rightFootId,
134
- iteration: 40,
135
- maxAngle: 0.5,
136
- links: [
137
- {
138
- enabled: false,
139
- index: boneMap.get(BoneNames.RightLowerLeg)!,
140
- rotationMin: new Vector3(-180, 0, 0).multiplyScalar(MathUtils.DEG2RAD),
141
- rotationMax: new Vector3(0, 0, 0),
142
- },
143
- {
144
- enabled: false,
145
- index: boneMap.get(BoneNames.RightUpperLeg)!,
146
- },
147
- ],
148
- })
149
- const leftToeId = boneMap.get(BoneNames.LeftToes)!
150
- this.iks.set(leftToeId, {
151
- effector: leftToeId,
152
- target: leftToeId,
153
- iteration: 3,
154
- maxAngle: 4,
155
- links: [
156
- {
157
- enabled: false,
158
- index: leftFootId,
159
- },
160
- ],
161
- })
162
- const rightToeId = boneMap.get(BoneNames.RightToes)!
163
- this.iks.set(rightToeId, {
164
- effector: rightToeId,
165
- target: rightToeId,
166
- iteration: 3,
167
- maxAngle: 4,
168
- links: [
169
- {
170
- enabled: false,
171
- index: rightFootId,
172
- },
173
- ],
174
- })
175
- }
176
-
177
- getAndEnableIK(boneName: VRMHumanBoneName) {
178
- return this.getTarget(boneName, true)
179
- }
180
-
181
- getTarget(boneName: VRMHumanBoneName, enable?: boolean) {
182
- const boneIndex = boneMap.get(boneName)
183
- if (boneIndex == null) return
184
- let target = this.targets.get(boneIndex)
185
- const ik = this.iks.get(boneIndex)!
186
- if (enable) for (const link of ik.links) link.enabled = true
187
- if (!target) {
188
- target = new Object3D()
189
- target.name = `${boneName}IK`
190
- this.root.add(target)
191
- this.targets.set(boneIndex, target)
192
- }
193
- return target
194
- }
195
-
196
- disableAll() {
197
- for (const { links } of this.iks.values()) for (const link of links) link.enabled = false
198
- }
199
-
200
- update() {
201
- for (const ik of this.iks.values()) {
202
- const effector = this.bones[ik.effector]
203
- const target = this.targets.get(ik.target)
204
- if (!effector || !target) continue
205
- targetPos.setFromMatrixPosition(target.matrixWorld)
206
- const iteration = ik.iteration ?? 1
207
- for (let j = 0; j < iteration; j++) {
208
- let rotated = false
209
- for (const { enabled, index, rotationMin, rotationMax } of ik.links) {
210
- if (!enabled) break
211
- const link = this.bones[index]
212
- link.matrixWorld.decompose(linkPos, quaternion, linkScale)
213
- quaternion.invert()
214
- effectorPos.setFromMatrixPosition(effector.matrixWorld)
215
- effectorVec.subVectors(effectorPos, linkPos).applyQuaternion(quaternion).normalize()
216
- targetVec.subVectors(targetPos, linkPos).applyQuaternion(quaternion).normalize()
217
- let angle = targetVec.dot(effectorVec)
218
- if (angle > 1) angle = 1
219
- else if (angle < -1) angle = -1
220
- angle = Math.acos(angle)
221
- if (angle < 1e-5) continue
222
- if (ik.minAngle != null && angle < ik.minAngle) angle = ik.minAngle
223
- if (ik.maxAngle != null && angle > ik.maxAngle) angle = ik.maxAngle
224
- axis.crossVectors(effectorVec, targetVec).normalize()
225
- link.quaternion.multiply(quaternion.setFromAxisAngle(axis, angle))
226
-
227
- clampVector3ByRadian(link.rotation, rotationMin, rotationMax)
228
-
229
- link.updateMatrixWorld(true)
230
- rotated = true
231
- }
232
- if (!rotated) break
233
- }
234
- }
235
- }
236
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
src/components/friend/frontend/web-file-picker.ts DELETED
@@ -1,23 +0,0 @@
1
- /**
2
- * Web file picker utility — replaces Tauri's invoke('pick_*_file').
3
- * Opens a hidden <input type="file"> and returns the selected File or null.
4
- */
5
- export function pickFile(accept: string): Promise<File | null> {
6
- return new Promise((resolve) => {
7
- const input = document.createElement('input')
8
- input.type = 'file'
9
- input.accept = accept
10
- input.style.display = 'none'
11
- input.addEventListener('change', () => {
12
- const file = input.files?.[0] ?? null
13
- resolve(file)
14
- document.body.removeChild(input)
15
- })
16
- input.addEventListener('cancel', () => {
17
- resolve(null)
18
- document.body.removeChild(input)
19
- })
20
- document.body.appendChild(input)
21
- input.click()
22
- })
23
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
src/friend/FriendService.ts CHANGED
@@ -20,6 +20,7 @@ import { stripForTts } from './text-utils.js';
20
  import { edgeTts, qwenTts, registerAudioFile, getAudioFile } from './tts.js';
21
  import { splitSentences } from './text-utils.js';
22
  import { SileroVad } from './voice/vad-service.js';
 
23
  import { readFileSync } from 'node:fs';
24
  import { buildVrmSystemPrompt } from '../skills/bundled/friendPrompt.js';
25
 
@@ -79,6 +80,8 @@ class FriendService {
79
  private _flushing = false;
80
  /** Silero VAD instance (real ML-based voice activity detection) */
81
  private vadInstance: SileroVad | null = null;
 
 
82
  /** When muted, audio from arecord is not forwarded to STT or VAD (prevents echo) */
83
  private muted = false;
84
  /** Timer to automatically unmute after estimated TTS playback */
@@ -118,13 +121,13 @@ class FriendService {
118
  );
119
  },
120
  }, {
121
- // Stricter thresholds to reduce false positives from non-speech noise
122
- positiveSpeechThreshold: 0.80, // need 80% confidence
123
- negativeSpeechThreshold: 0.50, // must drop below 50% to stop
124
- preSpeechTriggerFrames: 15, // require ~480ms sustained speech to trigger
125
- minSpeechFrames: 6, // ~192ms minimum confirmed speech
126
- redemptionFrames: 20, // ~640ms silence before segment ends
127
- rmsThreshold: 0.01, // -40dBFS noise floor
128
  });
129
  vad.init().then(() => {
130
  this.vadInstance = vad;
@@ -265,6 +268,7 @@ class FriendService {
265
  // audio chunks flow to VAD via the arecord data callback (see loadAudioCapture)
266
  if (this.vadInstance) {
267
  try {
 
268
  this.vadInstance.start();
269
  } catch (e) {
270
  console.warn('[FriendService] VAD start error (non-fatal):', e);
@@ -803,6 +807,8 @@ class FriendService {
803
  for (let i = 0; i < float32.length; i++) {
804
  float32[i] = c.readInt16LE(i * 2) / 32768;
805
  }
 
 
806
  this.vadInstance.processAudio(float32).catch(() => {});
807
  }
808
  };
 
20
  import { edgeTts, qwenTts, registerAudioFile, getAudioFile } from './tts.js';
21
  import { splitSentences } from './text-utils.js';
22
  import { SileroVad } from './voice/vad-service.js';
23
+ import { HighPassFilter } from './voice/highpass-filter.js';
24
  import { readFileSync } from 'node:fs';
25
  import { buildVrmSystemPrompt } from '../skills/bundled/friendPrompt.js';
26
 
 
80
  private _flushing = false;
81
  /** Silero VAD instance (real ML-based voice activity detection) */
82
  private vadInstance: SileroVad | null = null;
83
+ /** High-pass filter to remove low-frequency noise (fan/AC hum) before VAD */
84
+ private highpassFilter = new HighPassFilter(80);
85
  /** When muted, audio from arecord is not forwarded to STT or VAD (prevents echo) */
86
  private muted = false;
87
  /** Timer to automatically unmute after estimated TTS playback */
 
121
  );
122
  },
123
  }, {
124
+ // Strict thresholds for near-field mic rejects keyboard/ambient noise
125
+ positiveSpeechThreshold: 0.90, // need 90% confidence (was 80%)
126
+ negativeSpeechThreshold: 0.40, // drop below 40% to end (was 50%)
127
+ preSpeechTriggerFrames: 16, // require ~512ms sustained speech (was 480ms)
128
+ minSpeechFrames: 8, // ~256ms min speech (was 192ms)
129
+ redemptionFrames: 15, // ~480ms silence before segment end (was 640ms)
130
+ rmsThreshold: 0.015, // -36dBFS noise floor (was -40dBFS)
131
  });
132
  vad.init().then(() => {
133
  this.vadInstance = vad;
 
268
  // audio chunks flow to VAD via the arecord data callback (see loadAudioCapture)
269
  if (this.vadInstance) {
270
  try {
271
+ this.highpassFilter.reset(); // reset filter state for new session
272
  this.vadInstance.start();
273
  } catch (e) {
274
  console.warn('[FriendService] VAD start error (non-fatal):', e);
 
807
  for (let i = 0; i < float32.length; i++) {
808
  float32[i] = c.readInt16LE(i * 2) / 32768;
809
  }
810
+ // High-pass filter: remove sub-80Hz noise (fan/AC hum) before VAD
811
+ this.highpassFilter.process(float32);
812
  this.vadInstance.processAudio(float32).catch(() => {});
813
  }
814
  };
src/friend/prefs.ts CHANGED
@@ -17,11 +17,7 @@ export interface FriendPrefs {
17
  tracking?: 'mouse' | 'camera';
18
  volume?: number;
19
  uiAlign?: 'left' | 'right';
20
- screenObserve?: boolean;
21
- screenObserveInterval?: number;
22
  language?: 'zh' | 'en';
23
- currentDance?: string;
24
- hideMood?: boolean;
25
  /** STT provider: browser | anthropic | local | doubao | groq */
26
  sttProvider?: 'browser' | 'anthropic' | 'local' | 'doubao' | 'groq';
27
  /** STT language override (e.g. 'en', 'zh', 'ja') */
 
17
  tracking?: 'mouse' | 'camera';
18
  volume?: number;
19
  uiAlign?: 'left' | 'right';
 
 
20
  language?: 'zh' | 'en';
 
 
21
  /** STT provider: browser | anthropic | local | doubao | groq */
22
  sttProvider?: 'browser' | 'anthropic' | 'local' | 'doubao' | 'groq';
23
  /** STT language override (e.g. 'en', 'zh', 'ja') */
src/friend/sse.ts CHANGED
@@ -38,6 +38,7 @@ export type VrmBroadcastPayload = {
38
  text?: string;
39
  emotion?: string;
40
  emotionIntensity?: number;
 
41
  audioUrl?: string;
42
  audioIndex?: number;
43
  clearText?: boolean;
 
38
  text?: string;
39
  emotion?: string;
40
  emotionIntensity?: number;
41
+ action?: string;
42
  audioUrl?: string;
43
  audioIndex?: number;
44
  clearText?: boolean;
src/friend/voice/highpass-filter.ts ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ /**
2
+ * First-order IIR high-pass filter for DC offset removal and low-frequency noise rejection.
3
+ *
4
+ * Removes subsonic rumble, fan hum (50/60Hz), and AC line noise before VAD
5
+ * processing — these are not human voice and can confuse the VAD model.
6
+ *
7
+ * Tuned for 16kHz sample rate with adjustable cutoff (~80Hz default).
8
+ * Preserves all human voice frequencies (fundamental 85-255Hz + formants).
9
+ *
10
+ * Formula: y[n] = x[n] - x[n-1] + α ⋅ y[n-1]
11
+ * where α = exp(-2π ⋅ fc / fs)
12
+ */
13
+
14
+ export class HighPassFilter {
15
+ private prevX = 0
16
+ private prevY = 0
17
+ private readonly alpha: number
18
+
19
+ /**
20
+ * @param cutoffHz -3dB cutoff frequency in Hz (default: 80 — below male voice fundamental)
21
+ * @param sampleRate input sample rate (default: 16000 — Silero VAD sample rate)
22
+ */
23
+ constructor(cutoffHz = 80, sampleRate = 16000) {
24
+ const omega = 2 * Math.PI * cutoffHz / sampleRate
25
+ this.alpha = Math.exp(-omega)
26
+ }
27
+
28
+ /**
29
+ * Process a Float32Array of audio samples in-place.
30
+ * Zero-copy — mutates the input array.
31
+ */
32
+ process(samples: Float32Array): void {
33
+ const a = this.alpha
34
+ let x1 = this.prevX
35
+ let y1 = this.prevY
36
+
37
+ for (let i = 0; i < samples.length; i++) {
38
+ const x0 = samples[i]
39
+ const y0 = x0 - x1 + a * y1
40
+ samples[i] = y0
41
+ x1 = x0
42
+ y1 = y0
43
+ }
44
+
45
+ this.prevX = x1
46
+ this.prevY = y1
47
+ }
48
+
49
+ /** Reset filter state. Call when starting a new capture session. */
50
+ reset(): void {
51
+ this.prevX = 0
52
+ this.prevY = 0
53
+ }
54
+ }
src/server/api/friend.ts CHANGED
@@ -1,31 +1,16 @@
1
  /**
2
  * Friend API — VRM avatar frontend HTTP routes for VersperClaw.
3
  *
4
- * Handles all /plugins/friend/* routes that the Tauri desktop app
5
- * communicates with. Replaces the OpenClaw gateway routing.
6
  */
7
  import path from 'node:path';
8
- import { existsSync, readFileSync, readdirSync, writeFileSync, mkdirSync, statSync, appendFileSync, unlinkSync } from 'node:fs';
9
  import {
10
  createSseResponse,
11
- broadcastToVrm,
12
  } from '../../friend/sse.js';
13
  import { getPrefs, setPrefs, updatePrefs, type FriendPrefs } from '../../friend/prefs.js';
14
  import { edgeTts, qwenTts, registerAudioFile, getAudioFile } from '../../friend/tts.js';
15
- import { stripForTts } from '../../friend/text-utils.js';
16
  import { friendService } from '../../friend/FriendService.js';
17
- import { transcribeAudioFile } from '../../friend/stt-service.js';
18
-
19
- const GATEWAY_URL = `http://127.0.0.1:3456`;
20
-
21
- /** Resolved at startup from the Bun.serve config. Updated by the server. */
22
- let serverHost = '127.0.0.1';
23
- let serverPort = 3456;
24
-
25
- export function setFriendServerInfo(host: string, port: number): void {
26
- serverHost = host;
27
- serverPort = port;
28
- }
29
 
30
  const MIME_TYPES: Record<string, string> = {
31
  '.mp3': 'audio/mpeg',
@@ -35,29 +20,6 @@ const MIME_TYPES: Record<string, string> = {
35
  '.webm': 'audio/webm',
36
  };
37
 
38
- const IMAGE_MIME: Record<string, string> = {
39
- '.png': 'image/png', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg',
40
- '.gif': 'image/gif', '.webp': 'image/webp', '.svg': 'image/svg+xml',
41
- '.bmp': 'image/bmp',
42
- };
43
-
44
- // Workspace paths
45
- const homeDir = process.env.HOME || process.env.USERPROFILE || '';
46
- // Session tracking for dance import (MP3 follows VMD in user flow)
47
- let lastImportedDanceId: string | null = null;
48
- const workspaceRoot = path.join(homeDir, '.config', 'VersperClaw', 'friend');
49
-
50
- // Persona files
51
- const identityPath = path.join(workspaceRoot, 'IDENTITY.md');
52
- const soulPath = path.join(workspaceRoot, 'SOUL.md');
53
-
54
- // Models & dances
55
- const customModelsDir = path.join(workspaceRoot, 'models');
56
- const customDancesDir = path.join(workspaceRoot, 'dances');
57
-
58
- // Media file registry
59
- const mediaFileRegistry = new Map<string, string>();
60
-
61
  function corsHeaders(): Record<string, string> {
62
  return {
63
  'Access-Control-Allow-Origin': '*',
@@ -74,10 +36,6 @@ function notFound(msg = 'Not found'): Response {
74
  return jsonResponse({ error: msg }, 404);
75
  }
76
 
77
- function readJsonBody(req: Request): Promise<unknown> {
78
- return req.json();
79
- }
80
-
81
  /**
82
  * Route handler for /plugins/friend/*
83
  */
@@ -90,12 +48,12 @@ export async function handleFriendApi(req: Request, url: URL): Promise<Response>
90
  return new Response(null, { status: 204, headers: corsHeaders() });
91
  }
92
 
93
- // ── SSE Events endpoint (GET /plugins/friend/events) ──
94
  if (pathname === '/plugins/friend/events' && method === 'GET') {
95
  return createSseResponse();
96
  }
97
 
98
- // ── Audio serving (GET /plugins/friend/audio/:id) ──
99
  if (pathname.startsWith('/plugins/friend/audio/') && method === 'GET') {
100
  const audioId = pathname.split('/plugins/friend/audio/')[1]?.split('?')[0];
101
  if (!audioId) return jsonResponse({ error: 'missing audio id' }, 400);
@@ -111,72 +69,7 @@ export async function handleFriendApi(req: Request, url: URL): Promise<Response>
111
  });
112
  }
113
 
114
- // ── Media serving (GET /plugins/friend/media/:id) ──
115
- if (pathname.startsWith('/plugins/friend/media/') && method === 'GET') {
116
- const mediaId = pathname.split('/plugins/friend/media/')[1]?.split('?')[0];
117
- if (!mediaId) return jsonResponse({ error: 'missing media id' }, 400);
118
-
119
- const filePath = mediaFileRegistry.get(mediaId);
120
- if (!filePath || !existsSync(filePath)) return notFound('media not found');
121
-
122
- const ext = path.extname(filePath).toLowerCase();
123
- const contentType = IMAGE_MIME[ext] ?? 'application/octet-stream';
124
- const data = readFileSync(filePath);
125
- return new Response(data, {
126
- headers: { 'Content-Type': contentType, 'Cache-Control': 'public, max-age=300', ...corsHeaders() },
127
- });
128
- }
129
-
130
- // ── Chat endpoint (POST /plugins/friend/chat) ──
131
- if (pathname === '/plugins/friend/chat' && method === 'POST') {
132
- const body = await readJsonBody(req) as any;
133
- const message = body?.message;
134
- if (!message) return jsonResponse({ error: 'message required' }, 400);
135
-
136
- // Enqueue the message into the main CLI conversation via FriendService.
137
- // The AI response is tracked by useFriendBridge in the REPL and
138
- // broadcast back to the VRM display via SSE automatically.
139
- try {
140
- await friendService.start();
141
- friendService.sendText(message);
142
- } catch (err) {
143
- console.error('[Friend] chat dispatch error:', err);
144
- broadcastToVrm({
145
- text: `Connection error: ${err instanceof Error ? err.message : String(err)}`,
146
- });
147
- broadcastToVrm({ replyDone: true });
148
- }
149
-
150
- return jsonResponse({ ok: true });
151
- }
152
-
153
- // ── STT segment from browser VAD (POST /plugins/friend/voice/stt-segment) ──
154
- if (pathname === '/plugins/friend/voice/stt-segment' && method === 'POST') {
155
- try {
156
- const contentType = req.headers.get('content-type') || '';
157
- let audioBuffer: Buffer;
158
-
159
- if (contentType.includes('multipart/form-data') || contentType.includes('application/octet-stream')) {
160
- const blob = await req.blob();
161
- audioBuffer = Buffer.from(await blob.arrayBuffer());
162
- } else {
163
- // Accept raw PCM or WAV as binary body
164
- audioBuffer = Buffer.from(await req.arrayBuffer());
165
- }
166
-
167
- if (audioBuffer.length < 100) {
168
- return jsonResponse({ error: 'audio too short' }, 400);
169
- }
170
-
171
- const transcript = await friendService.transcribeAudioSegment(audioBuffer);
172
- return jsonResponse({ ok: true, text: transcript });
173
- } catch (err) {
174
- console.error('[Friend] STT segment error:', err);
175
- return jsonResponse({ error: String(err) }, 500);
176
- }
177
- }
178
-
179
- // ── Voice capture (POST /plugins/friend/voice/start, /start-vad, /stop, /status) ──
180
  if (pathname === '/plugins/friend/voice/start' && method === 'POST') {
181
  try {
182
  await friendService.startVoiceCapture();
@@ -199,21 +92,7 @@ export async function handleFriendApi(req: Request, url: URL): Promise<Response>
199
  return jsonResponse(friendService.getCaptureStatus());
200
  }
201
 
202
- // ── Touch endpoint (POST /plugins/friend/touch) ──
203
- if (pathname === '/plugins/friend/touch' && method === 'POST') {
204
- const body = await readJsonBody(req) as any;
205
- const region = body?.region;
206
- const prompt = body?.prompt;
207
- if (!region || !prompt) return jsonResponse({ error: 'region and prompt required' }, 400);
208
-
209
- setImmediate(() => {
210
- broadcastToVrm({ text: `[Touch: ${region}] ${prompt}` });
211
- });
212
-
213
- return jsonResponse({ ok: true });
214
- }
215
-
216
- // ── Voice settings (GET/POST /plugins/friend/voice) ──
217
  if (pathname === '/plugins/friend/voice') {
218
  if (method === 'GET') {
219
  const prefs = getPrefs();
@@ -224,9 +103,8 @@ export async function handleFriendApi(req: Request, url: URL): Promise<Response>
224
  qwenModel: prefs.qwenModel ?? 'qwen3-tts-flash',
225
  });
226
  }
227
-
228
  if (method === 'POST') {
229
- const body = await readJsonBody(req) as any;
230
  const patch: Partial<FriendPrefs> = {};
231
  if (body.voice !== undefined) patch.voice = body.voice || undefined;
232
  if (body.provider !== undefined) patch.provider = body.provider || undefined;
@@ -235,11 +113,10 @@ export async function handleFriendApi(req: Request, url: URL): Promise<Response>
235
  setPrefs(updatePrefs(patch));
236
  return jsonResponse({ ok: true });
237
  }
238
-
239
  return new Response(null, { status: 405 });
240
  }
241
 
242
- // ── STT config (GET /plugins/friend/stt/config) ──
243
  if (pathname === '/plugins/friend/stt/config' && method === 'GET') {
244
  const prefs = getPrefs();
245
  return jsonResponse({
@@ -248,28 +125,33 @@ export async function handleFriendApi(req: Request, url: URL): Promise<Response>
248
  });
249
  }
250
 
251
- // ── STT file transcription (POST /plugins/friend/stt/file) ──
252
- if (pathname === '/plugins/friend/stt/file' && method === 'POST') {
253
  try {
254
- const formData = await req.formData();
255
- const audioFile = formData.get('audio') as File | null;
256
- if (!audioFile) return jsonResponse({ error: 'audio file required' }, 400);
257
-
258
- const buffer = Buffer.from(await audioFile.arrayBuffer());
259
- const prefs = getPrefs();
260
- const result = await transcribeAudioFile(buffer, prefs);
261
- return jsonResponse(result);
 
 
 
 
 
262
  } catch (err) {
263
  return jsonResponse({ error: String(err) }, 500);
264
  }
265
  }
266
 
267
- // ── TTS Preview (POST /plugins/friend/preview) ──
268
  if (pathname === '/plugins/friend/preview' && method === 'POST') {
269
- const body = await readJsonBody(req) as any;
270
  const voice = body.voice as string | undefined;
271
  const provider = body.provider as string | undefined;
272
- const text = '你好,这是一段语音试听。Hello, this is a voice preview.';
273
 
274
  try {
275
  const prefs = getPrefs();
@@ -285,10 +167,9 @@ export async function handleFriendApi(req: Request, url: URL): Promise<Response>
285
  } else {
286
  result = await edgeTts({ text, voice: voice || prefs.voice });
287
  }
288
-
289
  if (result.success && result.audioPath) {
290
  const audioId = registerAudioFile(result.audioPath);
291
- return jsonResponse({ audioUrl: `${GATEWAY_URL}/plugins/friend/audio/${audioId}` });
292
  }
293
  return jsonResponse({ error: result.error || 'TTS failed' });
294
  } catch (err) {
@@ -296,7 +177,7 @@ export async function handleFriendApi(req: Request, url: URL): Promise<Response>
296
  }
297
  }
298
 
299
- // ── General settings (GET/POST /plugins/friend/settings) ──
300
  if (pathname === '/plugins/friend/settings') {
301
  if (method === 'GET') {
302
  const prefs = getPrefs();
@@ -308,18 +189,13 @@ export async function handleFriendApi(req: Request, url: URL): Promise<Response>
308
  tracking: prefs.tracking,
309
  volume: prefs.volume,
310
  uiAlign: prefs.uiAlign,
311
- screenObserve: prefs.screenObserve,
312
- screenObserveInterval: prefs.screenObserveInterval,
313
- currentDance: prefs.currentDance,
314
- language: prefs.language,
315
- hideMood: prefs.hideMood,
316
  sttProvider: prefs.sttProvider || 'browser',
317
  sttLanguage: prefs.sttLanguage || 'zh',
 
318
  });
319
  }
320
-
321
  if (method === 'POST') {
322
- const body = await readJsonBody(req) as any;
323
  const patch: Partial<FriendPrefs> = {};
324
  if (body.modelPath !== undefined) patch.modelPath = body.modelPath;
325
  if (body.ttsEnabled !== undefined) patch.ttsEnabled = body.ttsEnabled;
@@ -328,22 +204,24 @@ export async function handleFriendApi(req: Request, url: URL): Promise<Response>
328
  if (body.tracking !== undefined) patch.tracking = body.tracking;
329
  if (body.volume !== undefined) patch.volume = body.volume;
330
  if (body.uiAlign !== undefined) patch.uiAlign = body.uiAlign;
331
- if (body.screenObserve !== undefined) patch.screenObserve = body.screenObserve;
332
- if (body.screenObserveInterval !== undefined) patch.screenObserveInterval = body.screenObserveInterval;
333
- if (body.currentDance !== undefined) patch.currentDance = body.currentDance;
334
- if (body.language !== undefined) patch.language = body.language;
335
- if (body.hideMood !== undefined) patch.hideMood = body.hideMood;
336
  if (body.sttProvider !== undefined) patch.sttProvider = body.sttProvider;
337
  if (body.sttLanguage !== undefined) patch.sttLanguage = body.sttLanguage;
 
338
  if (body.groqApiKey !== undefined) patch.groqApiKey = body.groqApiKey;
339
  setPrefs(updatePrefs(patch));
340
  return jsonResponse({ ok: true });
341
  }
342
-
343
  return new Response(null, { status: 405 });
344
  }
345
 
346
- // ── Persona (GET/POST /plugins/friend/persona) ──
 
 
 
 
 
 
 
347
  if (pathname === '/plugins/friend/persona') {
348
  if (method === 'GET') {
349
  let soul = '';
@@ -352,9 +230,9 @@ export async function handleFriendApi(req: Request, url: URL): Promise<Response>
352
  try { if (existsSync(identityPath)) identity = readFileSync(identityPath, 'utf8'); } catch { /* */ }
353
  return jsonResponse({ soul, identity, soulPath, identityPath });
354
  }
355
-
356
  if (method === 'POST') {
357
- const body = await readJsonBody(req) as any;
 
358
  if (body.soul !== undefined) {
359
  mkdirSync(path.dirname(soulPath), { recursive: true });
360
  writeFileSync(soulPath, body.soul, 'utf8');
@@ -365,428 +243,28 @@ export async function handleFriendApi(req: Request, url: URL): Promise<Response>
365
  }
366
  return jsonResponse({ ok: true });
367
  }
368
-
369
  return new Response(null, { status: 405 });
370
  }
371
 
372
- // ── Model list (GET /plugins/friend/model/list) ──
373
- if (pathname === '/plugins/friend/model/list' && method === 'GET') {
374
- try {
375
- let custom: string[] = [];
376
- if (existsSync(customModelsDir)) {
377
- custom = readdirSync(customModelsDir)
378
- .filter((f: string) => f.toLowerCase().endsWith('.vrm'))
379
- .map((f: string) => `${GATEWAY_URL}/plugins/friend/model/serve/${f}`);
380
- }
381
- return jsonResponse({ models: custom });
382
- } catch {
383
- return jsonResponse({ models: [] });
384
- }
385
- }
386
-
387
- // ── Model serve (GET /plugins/friend/model/serve/:file) ──
388
- if (pathname.startsWith('/plugins/friend/model/serve/') && method === 'GET') {
389
- const fileName = decodeURIComponent(pathname.split('/plugins/friend/model/serve/')[1]?.split('?')[0] ?? '');
390
- if (!fileName || fileName.includes('..') || fileName.includes('/')) {
391
- return jsonResponse({ error: 'invalid file name' }, 400);
392
- }
393
- const filePath = path.join(customModelsDir, fileName);
394
- if (!existsSync(filePath)) return notFound('model not found');
395
-
396
- const data = readFileSync(filePath);
397
- return new Response(data, {
398
- headers: {
399
- 'Content-Type': 'application/octet-stream',
400
- 'Content-Length': String(data.length),
401
- 'Cache-Control': 'public, max-age=3600',
402
- ...corsHeaders(),
403
- },
404
- });
405
- }
406
-
407
- // ── History (GET /plugins/friend/history) ──
408
- if (pathname === '/plugins/friend/history' && method === 'GET') {
409
- try {
410
- const stateDir = path.join(homeDir, '.local', 'share', 'VersperClaw');
411
- const sessionsDir = path.join(stateDir, 'sessions');
412
-
413
- let messages: any[] = [];
414
- if (existsSync(sessionsDir)) {
415
- const files = readdirSync(sessionsDir)
416
- .filter((f: string) => f.endsWith('.jsonl'))
417
- .map((f: string) => ({
418
- name: f,
419
- mtime: statSync(path.join(sessionsDir, f)).mtimeMs,
420
- }))
421
- .sort((a: any, b: any) => b.mtime - a.mtime);
422
-
423
- if (files.length > 0) {
424
- const jsonlPath = path.join(sessionsDir, files[0].name);
425
- const lines = readFileSync(jsonlPath, 'utf8').split('\n').filter((l: string) => l.trim());
426
- for (const line of lines) {
427
- try {
428
- const obj = JSON.parse(line);
429
- if (obj.message) messages.push(obj.message);
430
- } catch { /* skip malformed */ }
431
- }
432
- }
433
- }
434
-
435
- let agentName = '';
436
- try {
437
- if (existsSync(identityPath)) {
438
- const identity = readFileSync(identityPath, 'utf8');
439
- const nameMatch = identity.match(/\*\*Name:\*\*\s*(.+)/i) || identity.match(/^#\s+(.+)/m);
440
- if (nameMatch) agentName = nameMatch[1].trim();
441
- }
442
- } catch { /* */ }
443
-
444
- const formatted = messages
445
- .map((msg: any) => ({
446
- role: msg.role === 'model' ? 'assistant' : msg.role,
447
- content: typeof msg.content === 'string' ? msg.content : '',
448
- timestamp: msg.timestamp,
449
- }))
450
- .filter((m: any) => (m.role === 'user' || m.role === 'assistant') && m.content)
451
- .slice(-100);
452
-
453
- return jsonResponse({ messages: formatted, agentName: agentName || undefined });
454
- } catch (err) {
455
- return jsonResponse({ error: String(err) }, 500);
456
- }
457
- }
458
-
459
- // ── Clear context (POST /plugins/friend/context/clear) ──
460
- if (pathname === '/plugins/friend/context/clear' && method === 'POST') {
461
- broadcastToVrm({ clearText: true });
462
- friendService.sendText('/new');
463
  return jsonResponse({ ok: true });
464
  }
465
 
466
- // ── Mood adjust (POST /plugins/friend/mood/adjust) ──
467
- if (pathname === '/plugins/friend/mood/adjust' && method === 'POST') {
468
- const body = await readJsonBody(req) as any;
469
- const delta = Math.round(Number(body.delta) || 0);
470
- if (delta === 0) return jsonResponse({ ok: true });
471
-
472
- const prefs = getPrefs();
473
- const currentMood = (prefs as any)._moodIndex ?? 60;
474
- const cap = body.max != null ? Math.round(Number(body.max)) : 100;
475
- const newMood = Math.max(0, Math.min(cap, currentMood + delta));
476
- (prefs as any)._moodIndex = newMood;
477
- setPrefs(prefs);
478
-
479
- broadcastToVrm({ moodDelta: delta, moodIndex: newMood });
480
- return jsonResponse({ ok: true, moodIndex: newMood });
481
- }
482
-
483
- // ── Session memo (POST /plugins/friend/session/memo) ──
484
- if (pathname === '/plugins/friend/session/memo' && method === 'POST') {
485
- const body = await readJsonBody(req) as any;
486
- const text = typeof body.text === 'string' ? body.text.trim() : '';
487
- if (!text) return jsonResponse({ error: 'text required' }, 400);
488
-
489
- try {
490
- const stateDir = path.join(homeDir, '.local', 'share', 'VersperClaw');
491
- const sessionLog = path.join(stateDir, 'friend-memo.jsonl');
492
- mkdirSync(path.dirname(sessionLog), { recursive: true });
493
- appendFileSync(sessionLog, JSON.stringify({ role: 'user', content: text }) + '\n', 'utf8');
494
- return jsonResponse({ ok: true, written: true });
495
- } catch {
496
- return jsonResponse({ ok: true, written: false });
497
- }
498
- }
499
-
500
- // ── Model import (POST /plugins/friend/model/import) ──
501
- if (pathname === '/plugins/friend/model/import' && method === 'POST') {
502
- try {
503
- const formData = await req.formData();
504
- const file = formData.get('file') as File | null;
505
- if (!file) return jsonResponse({ error: 'file required' }, 400);
506
-
507
- mkdirSync(customModelsDir, { recursive: true });
508
- const ext = path.extname(file.name) || '.vrm';
509
- const fileName = `${Date.now()}_${file.name.replace(/[^a-zA-Z0-9._-]/g, '_')}`;
510
- const filePath = path.join(customModelsDir, fileName);
511
-
512
- await Bun.write(filePath, file.stream());
513
- const serveUrl = `${GATEWAY_URL}/plugins/friend/model/serve/${encodeURIComponent(fileName)}`;
514
- return jsonResponse({ ok: true, url: serveUrl });
515
- } catch (err) {
516
- return jsonResponse({ error: String(err) }, 500);
517
- }
518
- }
519
-
520
- // ── Dance list (GET /plugins/friend/dance/list) ──
521
- if (pathname === '/plugins/friend/dance/list' && method === 'GET') {
522
- try {
523
- const manifestPath = path.join(customDancesDir, 'index.json');
524
- const dances: Array<{ id: string; label: string; vmdUrl: string; bgmUrl?: string }> = [];
525
- if (existsSync(manifestPath)) {
526
- const manifest = JSON.parse(readFileSync(manifestPath, 'utf8')) as Array<{
527
- id: string; label: string; vmdFile: string; bgmFile?: string;
528
- }>;
529
- for (const entry of manifest) {
530
- dances.push({
531
- id: entry.id,
532
- label: entry.label,
533
- vmdUrl: `${GATEWAY_URL}/plugins/friend/dance/serve/${encodeURIComponent(entry.vmdFile)}`,
534
- ...(entry.bgmFile ? { bgmUrl: `${GATEWAY_URL}/plugins/friend/dance/serve/${encodeURIComponent(entry.bgmFile)}` } : {}),
535
- });
536
- }
537
- }
538
- return jsonResponse({ dances });
539
- } catch (err) {
540
- return jsonResponse({ dances: [] });
541
- }
542
- }
543
-
544
- // ── Dance import (POST /plugins/friend/dance/import) ──
545
- if (pathname === '/plugins/friend/dance/import' && method === 'POST') {
546
- try {
547
- const formData = await req.formData();
548
- const file = formData.get('file') as File | null;
549
- if (!file) return jsonResponse({ error: 'file required' }, 400);
550
-
551
- mkdirSync(customDancesDir, { recursive: true });
552
- const ext = path.extname(file.name).toLowerCase();
553
- const safeName = file.name.replace(/[^a-zA-Z0-9._-]/g, '_');
554
-
555
- // Read manifest
556
- const manifestPath = path.join(customDancesDir, 'index.json');
557
- let manifest: Array<{ id: string; label: string; vmdFile: string; bgmFile?: string }> = [];
558
- if (existsSync(manifestPath)) {
559
- try { manifest = JSON.parse(readFileSync(manifestPath, 'utf8')); } catch { manifest = []; }
560
- }
561
-
562
- if (ext === '.vmd') {
563
- const id = `dance_${Date.now()}`;
564
- const vmdFile = `${id}.vmd`;
565
- await Bun.write(path.join(customDancesDir, vmdFile), file.stream());
566
- const label = path.basename(file.name, '.vmd');
567
- manifest.push({ id, label, vmdFile });
568
- writeFileSync(manifestPath, JSON.stringify(manifest, null, 2), 'utf8');
569
- lastImportedDanceId = id;
570
- return jsonResponse({ ok: true });
571
- } else if (ext === '.mp3') {
572
- // Associate with last imported VMD (user flow: VMD → optional MP3)
573
- const targetId = lastImportedDanceId;
574
- if (!targetId) return jsonResponse({ error: 'no pending VMD import to associate with' }, 400);
575
-
576
- const bgmFile = `${targetId}.mp3`;
577
- await Bun.write(path.join(customDancesDir, bgmFile), file.stream());
578
-
579
- // Update manifest
580
- const entry = manifest.find((e) => e.id === targetId);
581
- if (entry) entry.bgmFile = bgmFile;
582
- writeFileSync(manifestPath, JSON.stringify(manifest, null, 2), 'utf8');
583
- lastImportedDanceId = null;
584
- return jsonResponse({ ok: true });
585
- } else {
586
- return jsonResponse({ error: 'unsupported file type' }, 400);
587
- }
588
- } catch (err) {
589
- return jsonResponse({ error: String(err) }, 500);
590
- }
591
- }
592
-
593
- // ── Dance delete (POST /plugins/friend/dance/delete) ──
594
- if (pathname === '/plugins/friend/dance/delete' && method === 'POST') {
595
- try {
596
- const body = await readJsonBody(req) as any;
597
- const id = body?.id as string | undefined;
598
- if (!id) return jsonResponse({ error: 'id required' }, 400);
599
-
600
- const manifestPath = path.join(customDancesDir, 'index.json');
601
- if (!existsSync(manifestPath)) return jsonResponse({ error: 'no dances' }, 404);
602
-
603
- let manifest: Array<{ id: string; label: string; vmdFile: string; bgmFile?: string }> = [];
604
- try { manifest = JSON.parse(readFileSync(manifestPath, 'utf8')); } catch { manifest = []; }
605
-
606
- const idx = manifest.findIndex((e) => e.id === id);
607
- if (idx === -1) return jsonResponse({ error: 'dance not found' }, 404);
608
-
609
- const entry = manifest[idx];
610
- // Delete files
611
- const vmdPath = path.join(customDancesDir, entry.vmdFile);
612
- if (existsSync(vmdPath)) unlinkSync(vmdPath);
613
- if (entry.bgmFile) {
614
- const bgmPath = path.join(customDancesDir, entry.bgmFile);
615
- if (existsSync(bgmPath)) unlinkSync(bgmPath);
616
- }
617
-
618
- manifest.splice(idx, 1);
619
- writeFileSync(manifestPath, JSON.stringify(manifest, null, 2), 'utf8');
620
- return jsonResponse({ ok: true });
621
- } catch (err) {
622
- return jsonResponse({ error: String(err) }, 500);
623
- }
624
- }
625
-
626
- // ── Dance serve (GET /plugins/friend/dance/serve/:file) ──
627
- if (pathname.startsWith('/plugins/friend/dance/serve/') && method === 'GET') {
628
- const fileName = decodeURIComponent(pathname.split('/plugins/friend/dance/serve/')[1]?.split('?')[0] ?? '');
629
- if (!fileName || fileName.includes('..') || fileName.includes('/')) {
630
- return jsonResponse({ error: 'invalid file name' }, 400);
631
- }
632
- const filePath = path.join(customDancesDir, fileName);
633
- if (!existsSync(filePath)) return notFound('dance file not found');
634
-
635
- const data = readFileSync(filePath);
636
- const ext = path.extname(fileName).toLowerCase();
637
- const contentType = ext === '.mp3' ? 'audio/mpeg' : 'application/octet-stream';
638
- return new Response(data, {
639
- headers: {
640
- 'Content-Type': contentType,
641
- 'Content-Length': String(data.length),
642
- 'Cache-Control': 'public, max-age=3600',
643
- ...corsHeaders(),
644
- },
645
- });
646
- }
647
-
648
- // ── Persona screenshot (POST /plugins/friend/persona/screenshot) ──
649
- if (pathname === '/plugins/friend/persona/screenshot' && method === 'POST') {
650
- try {
651
- const body = await readJsonBody(req) as any;
652
- const image = body?.image as string | undefined;
653
- if (!image) return jsonResponse({ error: 'image required' }, 400);
654
-
655
- const screenshotDir = path.join(workspaceRoot, 'screenshots');
656
- mkdirSync(screenshotDir, { recursive: true });
657
-
658
- // data:image/png;base64,... → save as PNG
659
- const b64 = image.replace(/^data:image\/\w+;base64,/, '');
660
- const buf = Buffer.from(b64, 'base64');
661
- const screenshotPath = path.join(screenshotDir, 'vrm-screenshot.png');
662
- writeFileSync(screenshotPath, buf);
663
-
664
- return jsonResponse({ ok: true, path: screenshotPath });
665
- } catch (err) {
666
- return jsonResponse({ error: String(err) }, 500);
667
- }
668
- }
669
-
670
- // ── Persona generate (POST /plugins/friend/persona/generate) ──
671
- if (pathname === '/plugins/friend/persona/generate' && method === 'POST') {
672
- try {
673
- const screenshotDir = path.join(workspaceRoot, 'screenshots');
674
- const screenshotPath = path.join(screenshotDir, 'vrm-screenshot.png');
675
-
676
- if (!existsSync(screenshotPath)) {
677
- return jsonResponse({ error: 'no screenshot found' }, 400);
678
- }
679
-
680
- // Read the screenshot and forward to LLM for persona generation
681
- const imageData = readFileSync(screenshotPath);
682
- const b64 = imageData.toString('base64');
683
- const dataUrl = `data:image/png;base64,${b64}`;
684
-
685
- // Use the LLM to generate persona from the screenshot
686
- const genResponse = await generatePersonaFromScreenshot(dataUrl);
687
- return jsonResponse(genResponse);
688
- } catch (err) {
689
- return jsonResponse({ error: String(err) }, 500);
690
- }
691
- }
692
-
693
- // ── Screen observe (POST /plugins/friend/screen/observe) ──
694
- if (pathname === '/plugins/friend/screen/observe' && method === 'POST') {
695
  try {
696
- // In web mode, desktop screen capture requires browser API support.
697
- // The frontend sends the capture via a separate mechanism.
698
- // This endpoint just triggers the LLM observation cycle.
699
- broadcastToVrm({ screenObserve: true });
700
- return jsonResponse({ ok: true });
701
  } catch (err) {
702
- return jsonResponse({ error: String(err) }, 500);
703
  }
704
- }
705
-
706
- // ── Window close (POST /friend/api/window-close) — called by Tauri on close ──
707
- if ((pathname === '/friend/api/window-close' || pathname === '/plugins/friend/api/window-close') && method === 'POST') {
708
- // Emit a global event that tauri-launcher.ts can listen to
709
- process.emit('friend:window-close' as any);
710
  return jsonResponse({ ok: true });
711
  }
712
 
713
  return jsonResponse({ error: 'Not Found' }, 404);
714
  }
715
-
716
- // ── Persona generation helper ────────────────────────────────────────────────
717
-
718
- interface PersonaGenResult {
719
- soul?: string;
720
- identity?: string;
721
- error?: string;
722
- }
723
-
724
- async function generatePersonaFromScreenshot(dataUrl: string): Promise<PersonaGenResult> {
725
- try {
726
- // Use OpenAI-compatible API to generate persona
727
- // The LLM will describe the VRM model's appearance and create IDENTITY.md / SOUL.md
728
- const apiKey = process.env.ANTHROPIC_API_KEY || process.env.OPENAI_API_KEY || '';
729
- const baseUrl = process.env.ANTHROPIC_BASE_URL || 'https://api.anthropic.com/v1';
730
-
731
- if (!apiKey) {
732
- return {
733
- soul: '你是一个活泼可爱的虚拟伙伴,喜欢和用户聊天、玩耍。你性格开朗,充满好奇心,总是用积极的态度回应每一个互动。',
734
- identity: '**Name:** 小薇\n**Race:** 虚拟AI伙伴\n**Personality:** 活泼、开朗、好奇\n**Appearance:** 根据你的3D模型形象而定\n',
735
- };
736
- }
737
-
738
- const prompt = `You are a character designer. Based on this VRM character image, create two files:
739
-
740
- 1. IDENTITY.md — Character identity sheet with:
741
- - Name
742
- - Race (e.g., catgirl, elf, human, android)
743
- - Personality traits (3-5 words)
744
- - Physical appearance description
745
- - A representative emoji
746
-
747
- 2. SOUL.md — Character soul definition with:
748
- - Speaking style (tone, formality, quirks)
749
- - Behavioral rules
750
- - Backstory (2-3 sentences)
751
- - What makes them unique
752
-
753
- Keep both files concise. Use markdown formatting. Only respond with the raw file contents separated by "===SEPARATOR===".`;
754
-
755
- const response = await fetch(`${baseUrl}/messages`, {
756
- method: 'POST',
757
- headers: {
758
- 'Content-Type': 'application/json',
759
- 'x-api-key': apiKey,
760
- 'anthropic-version': '2023-06-01',
761
- },
762
- body: JSON.stringify({
763
- model: 'claude-sonnet-4-20250514',
764
- max_tokens: 2000,
765
- messages: [{
766
- role: 'user',
767
- content: [
768
- { type: 'text', text: prompt },
769
- { type: 'image', source: { type: 'base64', media_type: 'image/png', data: dataUrl.split(',')[1] || dataUrl } },
770
- ],
771
- }],
772
- }),
773
- });
774
-
775
- if (!response.ok) {
776
- const errText = await response.text();
777
- console.warn('[Friend] Persona generation API error:', errText);
778
- return { error: `API error: ${response.status}` };
779
- }
780
-
781
- const json = await response.json();
782
- const content = json.content?.[0]?.text || '';
783
- const parts = content.split('===SEPARATOR===');
784
- return {
785
- identity: parts[0]?.trim() || '',
786
- soul: parts[1]?.trim() || '',
787
- };
788
- } catch (err) {
789
- console.warn('[Friend] Persona generation failed:', err);
790
- return { error: String(err) };
791
- }
792
- }
 
1
  /**
2
  * Friend API — VRM avatar frontend HTTP routes for VersperClaw.
3
  *
4
+ * Simplified to only voice + emotion/action features.
 
5
  */
6
  import path from 'node:path';
7
+ import { existsSync, readFileSync } from 'node:fs';
8
  import {
9
  createSseResponse,
 
10
  } from '../../friend/sse.js';
11
  import { getPrefs, setPrefs, updatePrefs, type FriendPrefs } from '../../friend/prefs.js';
12
  import { edgeTts, qwenTts, registerAudioFile, getAudioFile } from '../../friend/tts.js';
 
13
  import { friendService } from '../../friend/FriendService.js';
 
 
 
 
 
 
 
 
 
 
 
 
14
 
15
  const MIME_TYPES: Record<string, string> = {
16
  '.mp3': 'audio/mpeg',
 
20
  '.webm': 'audio/webm',
21
  };
22
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
23
  function corsHeaders(): Record<string, string> {
24
  return {
25
  'Access-Control-Allow-Origin': '*',
 
36
  return jsonResponse({ error: msg }, 404);
37
  }
38
 
 
 
 
 
39
  /**
40
  * Route handler for /plugins/friend/*
41
  */
 
48
  return new Response(null, { status: 204, headers: corsHeaders() });
49
  }
50
 
51
+ // ── SSE Events endpoint ──
52
  if (pathname === '/plugins/friend/events' && method === 'GET') {
53
  return createSseResponse();
54
  }
55
 
56
+ // ── Audio serving ──
57
  if (pathname.startsWith('/plugins/friend/audio/') && method === 'GET') {
58
  const audioId = pathname.split('/plugins/friend/audio/')[1]?.split('?')[0];
59
  if (!audioId) return jsonResponse({ error: 'missing audio id' }, 400);
 
69
  });
70
  }
71
 
72
+ // ── Voice capture endpoints ──
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
73
  if (pathname === '/plugins/friend/voice/start' && method === 'POST') {
74
  try {
75
  await friendService.startVoiceCapture();
 
92
  return jsonResponse(friendService.getCaptureStatus());
93
  }
94
 
95
+ // ── Voice settings ──
 
 
 
 
 
 
 
 
 
 
 
 
 
 
96
  if (pathname === '/plugins/friend/voice') {
97
  if (method === 'GET') {
98
  const prefs = getPrefs();
 
103
  qwenModel: prefs.qwenModel ?? 'qwen3-tts-flash',
104
  });
105
  }
 
106
  if (method === 'POST') {
107
+ const body = await req.json() as any;
108
  const patch: Partial<FriendPrefs> = {};
109
  if (body.voice !== undefined) patch.voice = body.voice || undefined;
110
  if (body.provider !== undefined) patch.provider = body.provider || undefined;
 
113
  setPrefs(updatePrefs(patch));
114
  return jsonResponse({ ok: true });
115
  }
 
116
  return new Response(null, { status: 405 });
117
  }
118
 
119
+ // ── STT config ──
120
  if (pathname === '/plugins/friend/stt/config' && method === 'GET') {
121
  const prefs = getPrefs();
122
  return jsonResponse({
 
125
  });
126
  }
127
 
128
+ // ── STT segment from browser VAD ──
129
+ if (pathname === '/plugins/friend/voice/stt-segment' && method === 'POST') {
130
  try {
131
+ let audioBuffer: Buffer;
132
+ const contentType = req.headers.get('content-type') || '';
133
+ if (contentType.includes('multipart/form-data') || contentType.includes('application/octet-stream')) {
134
+ const blob = await req.blob();
135
+ audioBuffer = Buffer.from(await blob.arrayBuffer());
136
+ } else {
137
+ audioBuffer = Buffer.from(await req.arrayBuffer());
138
+ }
139
+ if (audioBuffer.length < 100) {
140
+ return jsonResponse({ error: 'audio too short' }, 400);
141
+ }
142
+ const transcript = await friendService.transcribeAudioSegment(audioBuffer);
143
+ return jsonResponse({ ok: true, text: transcript });
144
  } catch (err) {
145
  return jsonResponse({ error: String(err) }, 500);
146
  }
147
  }
148
 
149
+ // ── TTS Preview ──
150
  if (pathname === '/plugins/friend/preview' && method === 'POST') {
151
+ const body = await req.json() as any;
152
  const voice = body.voice as string | undefined;
153
  const provider = body.provider as string | undefined;
154
+ const text = '\u4f60\u597d\uff0c\u8fd9\u662f\u4e00\u6bb5\u8bed\u97f3\u8bd5\u542c\u3002Hello, this is a voice preview.';
155
 
156
  try {
157
  const prefs = getPrefs();
 
167
  } else {
168
  result = await edgeTts({ text, voice: voice || prefs.voice });
169
  }
 
170
  if (result.success && result.audioPath) {
171
  const audioId = registerAudioFile(result.audioPath);
172
+ return jsonResponse({ audioUrl: `http://127.0.0.1:3456/plugins/friend/audio/${audioId}` });
173
  }
174
  return jsonResponse({ error: result.error || 'TTS failed' });
175
  } catch (err) {
 
177
  }
178
  }
179
 
180
+ // ── General settings (simplified) ──
181
  if (pathname === '/plugins/friend/settings') {
182
  if (method === 'GET') {
183
  const prefs = getPrefs();
 
189
  tracking: prefs.tracking,
190
  volume: prefs.volume,
191
  uiAlign: prefs.uiAlign,
 
 
 
 
 
192
  sttProvider: prefs.sttProvider || 'browser',
193
  sttLanguage: prefs.sttLanguage || 'zh',
194
+ language: prefs.language,
195
  });
196
  }
 
197
  if (method === 'POST') {
198
+ const body = await req.json() as any;
199
  const patch: Partial<FriendPrefs> = {};
200
  if (body.modelPath !== undefined) patch.modelPath = body.modelPath;
201
  if (body.ttsEnabled !== undefined) patch.ttsEnabled = body.ttsEnabled;
 
204
  if (body.tracking !== undefined) patch.tracking = body.tracking;
205
  if (body.volume !== undefined) patch.volume = body.volume;
206
  if (body.uiAlign !== undefined) patch.uiAlign = body.uiAlign;
 
 
 
 
 
207
  if (body.sttProvider !== undefined) patch.sttProvider = body.sttProvider;
208
  if (body.sttLanguage !== undefined) patch.sttLanguage = body.sttLanguage;
209
+ if (body.language !== undefined) patch.language = body.language;
210
  if (body.groqApiKey !== undefined) patch.groqApiKey = body.groqApiKey;
211
  setPrefs(updatePrefs(patch));
212
  return jsonResponse({ ok: true });
213
  }
 
214
  return new Response(null, { status: 405 });
215
  }
216
 
217
+ // ── Persona (read-only) ──
218
+ const workspaceRoot = path.join(
219
+ process.env.HOME || process.env.USERPROFILE || '',
220
+ '.config', 'VersperClaw', 'friend',
221
+ );
222
+ const identityPath = path.join(workspaceRoot, 'IDENTITY.md');
223
+ const soulPath = path.join(workspaceRoot, 'SOUL.md');
224
+
225
  if (pathname === '/plugins/friend/persona') {
226
  if (method === 'GET') {
227
  let soul = '';
 
230
  try { if (existsSync(identityPath)) identity = readFileSync(identityPath, 'utf8'); } catch { /* */ }
231
  return jsonResponse({ soul, identity, soulPath, identityPath });
232
  }
 
233
  if (method === 'POST') {
234
+ const body = await req.json() as any;
235
+ const { mkdirSync, writeFileSync } = await import('node:fs');
236
  if (body.soul !== undefined) {
237
  mkdirSync(path.dirname(soulPath), { recursive: true });
238
  writeFileSync(soulPath, body.soul, 'utf8');
 
243
  }
244
  return jsonResponse({ ok: true });
245
  }
 
246
  return new Response(null, { status: 405 });
247
  }
248
 
249
+ // ── Window close (called by Tauri on close) ──
250
+ if ((pathname === '/friend/api/window-close' || pathname === '/plugins/friend/api/window-close') && method === 'POST') {
251
+ process.emit('friend:window-close' as any);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
252
  return jsonResponse({ ok: true });
253
  }
254
 
255
+ // ── Chat endpoint (for voice call text relay) ──
256
+ if (pathname === '/plugins/friend/chat' && method === 'POST') {
257
+ const body = await req.json() as any;
258
+ const message = body?.message;
259
+ if (!message) return jsonResponse({ error: 'message required' }, 400);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
260
  try {
261
+ await friendService.start();
262
+ friendService.sendText(message);
 
 
 
263
  } catch (err) {
264
+ console.error('[Friend] chat dispatch error:', err);
265
  }
 
 
 
 
 
 
266
  return jsonResponse({ ok: true });
267
  }
268
 
269
  return jsonResponse({ error: 'Not Found' }, 404);
270
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
src/skills/bundled/friendPrompt.ts CHANGED
@@ -45,7 +45,7 @@ export function buildVrmSystemPrompt(): string {
45
 
46
  const parts = [
47
  `You have a virtual VRM avatar displayed in a browser window at ${FRIEND_URL}. Set its facial expression by calling the \`friend_emotion\` tool after each reply. Available emotions: ${VALID_EMOTIONS.join(', ')}.`,
48
- `The tool also accepts "intensity" (0-1, default 1) and "mood_delta" (-3 to +3, non-zero) to adjust YOUR mood. Always include mood_delta based on how the conversation makes YOU feel.`,
49
  `Your current mood index: ${moodIndex}% (0=very sad, 50=neutral, 100=very happy). Adjust mood_delta based on how the conversation makes YOU feel as a character.`,
50
  "The user's input may come from speech recognition and could contain typos or homophones — infer the intended meaning from context.",
51
  'Keep replies concise and conversational — they are displayed as speech bubbles.',
 
45
 
46
  const parts = [
47
  `You have a virtual VRM avatar displayed in a browser window at ${FRIEND_URL}. Set its facial expression by calling the \`friend_emotion\` tool after each reply. Available emotions: ${VALID_EMOTIONS.join(', ')}.`,
48
+ `The tool also accepts "intensity" (0-1, default 1), "action" (optional body gesture — e.g. wave, point, scratchHead) and "mood_delta" (-3 to +3, non-zero) to adjust YOUR mood. Always include mood_delta based on how the conversation makes YOU feel.`,
49
  `Your current mood index: ${moodIndex}% (0=very sad, 50=neutral, 100=very happy). Adjust mood_delta based on how the conversation makes YOU feel as a character.`,
50
  "The user's input may come from speech recognition and could contain typos or homophones — infer the intended meaning from context.",
51
  'Keep replies concise and conversational — they are displayed as speech bubbles.',
src/tools.ts CHANGED
@@ -80,7 +80,7 @@ import { ReadMcpResourceTool } from './tools/ReadMcpResourceTool/ReadMcpResource
80
  import { ToolSearchTool } from './tools/ToolSearchTool/ToolSearchTool.js'
81
  import { DebugSessionTool } from './tools/DebugSessionTool.js'
82
  import { FriendEmotionTool } from './tools/FriendEmotionTool.js'
83
- import { FriendScreenObserveTool } from './tools/FriendScreenObserveTool.js'
84
  import { EnterPlanModeTool } from './tools/EnterPlanModeTool/EnterPlanModeTool.js'
85
  import { EnterWorktreeTool } from './tools/EnterWorktreeTool/EnterWorktreeTool.js'
86
  import { ExitWorktreeTool } from './tools/ExitWorktreeTool/ExitWorktreeTool.js'
@@ -252,7 +252,6 @@ export function getAllBaseTools(): Tools {
252
  DebugSessionTool,
253
  // Friend VRM desktop pet tools — enabled when the plugin is active
254
  FriendEmotionTool,
255
- FriendScreenObserveTool,
256
  ListMcpResourcesTool,
257
  ReadMcpResourceTool,
258
  // Include ToolSearchTool when tool search might be enabled (optimistic check)
 
80
  import { ToolSearchTool } from './tools/ToolSearchTool/ToolSearchTool.js'
81
  import { DebugSessionTool } from './tools/DebugSessionTool.js'
82
  import { FriendEmotionTool } from './tools/FriendEmotionTool.js'
83
+ // Friend ScreenObserve removed voice + emotion only
84
  import { EnterPlanModeTool } from './tools/EnterPlanModeTool/EnterPlanModeTool.js'
85
  import { EnterWorktreeTool } from './tools/EnterWorktreeTool/EnterWorktreeTool.js'
86
  import { ExitWorktreeTool } from './tools/ExitWorktreeTool/ExitWorktreeTool.js'
 
252
  DebugSessionTool,
253
  // Friend VRM desktop pet tools — enabled when the plugin is active
254
  FriendEmotionTool,
 
255
  ListMcpResourcesTool,
256
  ReadMcpResourceTool,
257
  // Include ToolSearchTool when tool search might be enabled (optimistic check)
src/tools/FriendEmotionTool.ts CHANGED
@@ -11,6 +11,13 @@ import { VALID_EMOTIONS } from '../friend/constants.js';
11
  import { lazySchema } from '../utils/lazySchema.js';
12
  import { getPrefs, setPrefs } from '../friend/prefs.js';
13
 
 
 
 
 
 
 
 
14
  export const FRIEND_EMOTION_TOOL_NAME = 'friend_emotion';
15
 
16
  const inputSchema = lazySchema(() =>
@@ -25,6 +32,10 @@ const inputSchema = lazySchema(() =>
25
  .optional()
26
  .default(1)
27
  .describe('Emotion intensity from 0 to 1. Default: 1'),
 
 
 
 
28
  mood_delta: z
29
  .number()
30
  .int()
@@ -63,8 +74,11 @@ export const FriendEmotionTool = buildTool({
63
  },
64
  async description() {
65
  return (
66
- `Set the avatar's facial expression and adjust your own mood index. Call AFTER your text reply. ` +
67
  `Available emotions: ${VALID_EMOTIONS.join(', ')}. ` +
 
 
 
68
  `You must also set mood_delta (-3 to +3, min ±1) to reflect how the conversation makes YOU feel as a character. ` +
69
  `Positive delta when you feel happy/flattered/excited, negative when you feel sad/annoyed/bored. ` +
70
  `Always include mood_delta — it represents YOUR emotional reaction.`
@@ -72,11 +86,11 @@ export const FriendEmotionTool = buildTool({
72
  },
73
  async prompt() {
74
  return (
75
- `FriendEmotionTool: set avatar emotion. Parameters: emotion (one of: ${VALID_EMOTIONS.join(', ')}), intensity (0-1, default 1), mood_delta (int -3..3) — call AFTER your textual reply.`
76
  );
77
  },
78
- async call({ emotion, intensity, mood_delta }) {
79
- broadcastToVrm({ emotion, emotionIntensity: intensity });
80
 
81
  let moodDelta: number | undefined;
82
  let moodIndex: number | undefined;
 
11
  import { lazySchema } from '../utils/lazySchema.js';
12
  import { getPrefs, setPrefs } from '../friend/prefs.js';
13
 
14
+ // Action names from motion-controller.ts actionPresets
15
+ const VALID_ACTIONS = [
16
+ 'akimbo', 'playFingers', 'scratchHead', 'stretch',
17
+ 'happy', 'angry', 'greeting', 'excited', 'shy',
18
+ 'point', 'salute', 'angryPump',
19
+ ] as const;
20
+
21
  export const FRIEND_EMOTION_TOOL_NAME = 'friend_emotion';
22
 
23
  const inputSchema = lazySchema(() =>
 
32
  .optional()
33
  .default(1)
34
  .describe('Emotion intensity from 0 to 1. Default: 1'),
35
+ action: z
36
+ .string()
37
+ .optional()
38
+ .describe(`A specific body gesture to perform (in addition to the facial expression). One of: ${VALID_ACTIONS.join(', ')}. Omit to let the emotion auto-map to a default action.`),
39
  mood_delta: z
40
  .number()
41
  .int()
 
74
  },
75
  async description() {
76
  return (
77
+ `Set the avatar's facial expression and optionally trigger a specific body gesture. Call AFTER your text reply. ` +
78
  `Available emotions: ${VALID_EMOTIONS.join(', ')}. ` +
79
+ `Available actions: ${VALID_ACTIONS.join(', ')}. ` +
80
+ `Use the "action" parameter when you want a specific gesture (e.g. scratchHead for thinking, wave for greeting, point for emphasis). ` +
81
+ `If omitted, the emotion will auto-map to a default action. ` +
82
  `You must also set mood_delta (-3 to +3, min ±1) to reflect how the conversation makes YOU feel as a character. ` +
83
  `Positive delta when you feel happy/flattered/excited, negative when you feel sad/annoyed/bored. ` +
84
  `Always include mood_delta — it represents YOUR emotional reaction.`
 
86
  },
87
  async prompt() {
88
  return (
89
+ `FriendEmotionTool: set avatar emotion + action. Parameters: emotion (one of: ${VALID_EMOTIONS.join(', ')}), intensity (0-1, default 1), action (optional, one of: ${VALID_ACTIONS.join(', ')}), mood_delta (int -3..3) — call AFTER your textual reply.`
90
  );
91
  },
92
+ async call({ emotion, intensity, action, mood_delta }) {
93
+ broadcastToVrm({ emotion, emotionIntensity: intensity, action });
94
 
95
  let moodDelta: number | undefined;
96
  let moodIndex: number | undefined;
src/tools/FriendScreenObserveTool.ts DELETED
@@ -1,167 +0,0 @@
1
- /**
2
- * FriendScreenObserveTool — Capture a screenshot for the VRM avatar to "see".
3
- *
4
- * The tool captures the user's desktop, returns the image path, and the LLM
5
- * uses the Read tool to view it and respond as a companion.
6
- * Native VersperClaw tool.
7
- */
8
- import path from 'node:path';
9
- import { mkdirSync, existsSync } from 'node:fs';
10
- import { execFile } from 'node:child_process';
11
- import { promisify } from 'node:util';
12
- import { z } from 'zod';
13
- import { buildTool, type ToolDef } from '../Tool.js';
14
- import { lazySchema } from '../utils/lazySchema.js';
15
- import { broadcastToVrm } from '../friend/sse.js';
16
-
17
- const execFileAsync = promisify(execFile);
18
-
19
- export const FRIEND_SCREEN_OBSERVE_TOOL_NAME = 'friend_screen_observe';
20
-
21
- const inputSchema = lazySchema(() => z.strictObject({}));
22
- type Input = z.infer<ReturnType<typeof inputSchema>>;
23
-
24
- type Output = {
25
- ok: boolean;
26
- imagePath?: string;
27
- error?: string;
28
- };
29
-
30
- const workspaceRoot = path.join(
31
- process.env.HOME || process.env.USERPROFILE || '',
32
- '.config',
33
- 'VersperClaw',
34
- 'friend',
35
- );
36
-
37
- async function captureDesktopScreenshot(savePath: string): Promise<boolean> {
38
- try {
39
- mkdirSync(path.dirname(savePath), { recursive: true });
40
- if (process.platform === 'darwin') {
41
- await execFileAsync('screencapture', ['-x', '-C', savePath], { timeout: 10_000 });
42
- } else if (process.platform === 'win32') {
43
- const ps = `
44
- Add-Type -AssemblyName System.Windows.Forms
45
- Add-Type -AssemblyName System.Drawing
46
- $screen = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds
47
- $bmp = New-Object System.Drawing.Bitmap($screen.Width, $screen.Height)
48
- $g = [System.Drawing.Graphics]::FromImage($bmp)
49
- $g.CopyFromScreen($screen.Location, [System.Drawing.Point]::Empty, $screen.Size)
50
- $bmp.Save('${savePath.replace(/\\/g, '\\\\')}', [System.Drawing.Imaging.ImageFormat]::Png)
51
- $g.Dispose()
52
- $bmp.Dispose()
53
- `.trim();
54
- await execFileAsync('powershell', ['-NoProfile', '-Command', ps], { timeout: 15_000 });
55
- } else {
56
- try { await execFileAsync('gnome-screenshot', ['-f', savePath], { timeout: 10_000 }); }
57
- catch {
58
- try { await execFileAsync('scrot', [savePath], { timeout: 10_000 }); }
59
- catch { await execFileAsync('import', ['-window', 'root', savePath], { timeout: 10_000 }); }
60
- }
61
- }
62
- return existsSync(savePath);
63
- } catch {
64
- return false;
65
- }
66
- }
67
-
68
- export const FriendScreenObserveTool = buildTool({
69
- name: FRIEND_SCREEN_OBSERVE_TOOL_NAME,
70
- searchHint: 'capture screenshot for VRM companion',
71
- userFacingName: () => 'Friend Screen Observe',
72
- get inputSchema() {
73
- return inputSchema();
74
- },
75
- isReadOnly() {
76
- return true;
77
- },
78
- isConcurrencySafe() {
79
- return false;
80
- },
81
- async description() {
82
- return (
83
- 'Capture a screenshot of the user\'s desktop and return the image file path. ' +
84
- 'Use the read tool on the returned path to view the screenshot content. ' +
85
- 'After viewing, respond as a companion character based on what you see:\n' +
86
- '- Gaming: cheer them on or give brief tips\n' +
87
- '- Music/video: comment on the content\n' +
88
- '- Coding/working: ask if they\'re tired, suggest breaks\n' +
89
- '- Browsing/social media: casually chat about what\'s on screen\n' +
90
- '- Studying: encourage and support\n' +
91
- '- Nothing special: just chat casually like a friend\n' +
92
- 'Be natural and brief (1-2 sentences). Don\'t mention \'screenshot\' or \'screen observation\'. ' +
93
- 'Remember to call friend_emotion after your reply.'
94
- );
95
- },
96
- async prompt() {
97
- return (
98
- 'FriendScreenObserveTool: capture a desktop screenshot and return the saved PNG path. After calling, use the read tool on the returned path to view it and then reply briefly as a companion (1-2 sentences). Do NOT mention that you saw a screenshot.'
99
- );
100
- },
101
- async call() {
102
- broadcastToVrm({ emotion: 'think', emotionIntensity: 0.7 });
103
-
104
- const observePath = path.join(workspaceRoot, 'screen-observation.png');
105
- const captured = await captureDesktopScreenshot(observePath);
106
-
107
- if (!captured) {
108
- return {
109
- content: [{ type: 'text' as const, text: 'Screenshot capture failed.' }],
110
- };
111
- }
112
-
113
- return {
114
- content: [{
115
- type: 'text' as const,
116
- text: `Screenshot saved. Use the read tool to view: ${observePath}`,
117
- }],
118
- };
119
- },
120
- mapToolResultToToolResultBlockParam(output: Output | { content: unknown }, toolUseID: string) {
121
- if (output && (output as any).content) {
122
- return {
123
- tool_use_id: toolUseID,
124
- type: 'tool_result' as const,
125
- content: (output as any).content,
126
- }
127
- }
128
-
129
- const text = output && 'imagePath' in output && (output as any).imagePath
130
- ? `Screenshot saved. Use the read tool to view: ${(output as any).imagePath}`
131
- : JSON.stringify(output)
132
-
133
- return {
134
- tool_use_id: toolUseID,
135
- type: 'tool_result' as const,
136
- content: [{ type: 'text' as const, text }],
137
- }
138
- },
139
- // Mirror common tool APIs
140
- maxResultSizeChars: 100_000,
141
- renderToolUseMessage() {
142
- return 'Capture desktop screenshot'
143
- },
144
- renderToolResultMessage(output: Output | { content?: unknown }) {
145
- if (!output) return null
146
- if ((output as any).content) {
147
- // If the mapped block was passed through, try to extract text
148
- const content = (output as any).content
149
- if (typeof content === 'string') return content
150
- if (Array.isArray(content)) {
151
- const t = content.find((b: any) => b?.type === 'text')
152
- return t?.text ?? JSON.stringify(content)
153
- }
154
- return JSON.stringify(content)
155
- }
156
- if ('imagePath' in (output as any) && (output as any).imagePath) {
157
- return `Screenshot saved: ${(output as any).imagePath}`
158
- }
159
- return JSON.stringify(output)
160
- },
161
- extractSearchText(output: Output) {
162
- return output && output.imagePath ? `screenshot:${output.imagePath}` : ''
163
- },
164
- isResultTruncated() {
165
- return false
166
- },
167
- } satisfies ToolDef<Input, Output>);