feat: /friend aliveness system, motion controller fix, VAD tuning, high-pass filter
Browse files- Remove redundant files (HistoryPanel, MoodIndicator, vmd-loader, vrm-ik-handler, web-file-picker, FriendScreenObserveTool)
- Add action parameter to friend_emotion for VRM animation playback
- Fix motion-controller: replace broken 'finished' event with frame-accurate completion detection and action queue
- Add aliveness system: breathing, postural sway, speech micro-movements, listening response, micro-expressions, enhanced blink
- Add Silero VAD threshold tuning and high-pass filter for low-frequency noise rejection
- Fix build scripts (vite build instead of tauri build)
- Add LipSync.isActive() for speech-driven animation blending
Co-Authored-By: Claude Big Pickle <noreply@anthropic.com>
- package.json +2 -1
- src/components/friend/frontend/App.tsx +45 -242
- src/components/friend/frontend/components/ChatInput.tsx +81 -340
- src/components/friend/frontend/components/HistoryPanel.tsx +0 -177
- src/components/friend/frontend/components/MoodIndicator.tsx +0 -372
- src/components/friend/frontend/components/SettingsPanel.tsx +93 -489
- src/components/friend/frontend/components/TextBubble.tsx +1 -0
- src/components/friend/frontend/components/VRMScene.tsx +128 -301
- src/components/friend/frontend/index.html +0 -39
- src/components/friend/frontend/lip-sync.ts +5 -0
- src/components/friend/frontend/mmd-parser.d.ts +0 -39
- src/components/friend/frontend/motion-controller.ts +69 -174
- src/components/friend/frontend/vmd-loader.ts +0 -663
- src/components/friend/frontend/vrm-ik-handler.ts +0 -236
- src/components/friend/frontend/web-file-picker.ts +0 -23
- src/friend/FriendService.ts +13 -7
- src/friend/prefs.ts +0 -4
- src/friend/sse.ts +1 -0
- src/friend/voice/highpass-filter.ts +54 -0
- src/server/api/friend.ts +52 -574
- src/skills/bundled/friendPrompt.ts +1 -1
- src/tools.ts +1 -2
- src/tools/FriendEmotionTool.ts +18 -4
- src/tools/FriendScreenObserveTool.ts +0 -167
package.json
CHANGED
|
@@ -18,7 +18,8 @@
|
|
| 18 |
"scripts": {
|
| 19 |
"build": "bun run ./scripts/build.ts",
|
| 20 |
"dev": "bun run ./scripts/dev.ts",
|
| 21 |
-
"friend:build": "cd src/components/friend/frontend && npx
|
|
|
|
| 22 |
},
|
| 23 |
"dependencies": {
|
| 24 |
"@alcalzone/ansi-tokenize": "^0.3.0",
|
|
|
|
| 18 |
"scripts": {
|
| 19 |
"build": "bun run ./scripts/build.ts",
|
| 20 |
"dev": "bun run ./scripts/dev.ts",
|
| 21 |
+
"friend:build": "cd src/components/friend/frontend && npx vite build",
|
| 22 |
+
"friend:dev": "cd src/components/friend/frontend && npx vite"
|
| 23 |
},
|
| 24 |
"dependencies": {
|
| 25 |
"@alcalzone/ansi-tokenize": "^0.3.0",
|
src/components/friend/frontend/App.tsx
CHANGED
|
@@ -1,27 +1,19 @@
|
|
| 1 |
import { useEffect, useRef, useState, useCallback } from 'react'
|
| 2 |
import { VRMScene } from './components/VRMScene'
|
| 3 |
-
import type { VRMSceneHandle
|
| 4 |
import { TextBubble } from './components/TextBubble'
|
| 5 |
import type { OnVrmMessage } from './components/TextBubble'
|
| 6 |
import { ChatInput } from './components/ChatInput'
|
| 7 |
import { ResizeHandles } from './components/ResizeHandles'
|
| 8 |
import { SettingsPanel } from './components/SettingsPanel'
|
| 9 |
import { usePassThrough } from './hooks/usePassThrough'
|
| 10 |
-
import { dancePresets } from './motion-controller'
|
| 11 |
import { LipSync } from './lip-sync'
|
| 12 |
import { FRIEND_API, bindScene } from './api'
|
| 13 |
-
import {
|
| 14 |
-
import { MoodIndicator } from './components/MoodIndicator'
|
| 15 |
-
import { Menu, Pin, Move, RotateCcw, Rotate3D, EyeOff, Settings, Music, RefreshCw } from 'lucide-react'
|
| 16 |
|
| 17 |
const DEFAULT_MODEL = '/friend/model1.vrm'
|
| 18 |
-
// Module-level to survive any component remount
|
| 19 |
-
let lastTouchMemoTime = 0
|
| 20 |
-
const TOUCH_MEMO_COOLDOWN = 3_000
|
| 21 |
-
let lastTouchChatTime = 0
|
| 22 |
-
const TOUCH_CHAT_COOLDOWN = 60_000
|
| 23 |
|
| 24 |
-
// 情绪 → 动作映射
|
| 25 |
const emotionActionMap: Record<string, string> = {
|
| 26 |
think: 'scratchHead',
|
| 27 |
question: 'point',
|
|
@@ -38,7 +30,6 @@ const emotionActionMap: Record<string, string> = {
|
|
| 38 |
neutral: 'salute',
|
| 39 |
}
|
| 40 |
|
| 41 |
-
|
| 42 |
const btnStyle: React.CSSProperties = {
|
| 43 |
width: 32,
|
| 44 |
height: 32,
|
|
@@ -60,24 +51,16 @@ export default function App() {
|
|
| 60 |
const [tracking, setTracking] = useState<'mouse' | 'camera'>('mouse')
|
| 61 |
const [showText, setShowText] = useState(true)
|
| 62 |
const [collapsed, setCollapsed] = useState(false)
|
| 63 |
-
|
| 64 |
const [ttsEnabled, setTtsEnabled] = useState(true)
|
| 65 |
const [modelPath, setModelPath] = useState(DEFAULT_MODEL)
|
| 66 |
const [settingsOpen, setSettingsOpen] = useState(false)
|
| 67 |
-
const [historyOpen, setHistoryOpen] = useState(false)
|
| 68 |
const [hideUI, setHideUI] = useState(false)
|
| 69 |
const [volume, setVolume] = useState(0.5)
|
| 70 |
const [uiAlign, setUiAlign] = useState<'left' | 'right'>('right')
|
| 71 |
-
const [dancing, setDancing] = useState(false)
|
| 72 |
-
const [currentDance, setCurrentDance] = useState('jile')
|
| 73 |
-
const [customDancePreset, setCustomDancePreset] = useState<import('./motion-controller').DancePreset | undefined>(undefined)
|
| 74 |
-
const [hideMood, setHideMood] = useState(false)
|
| 75 |
-
const [screenObserve, setScreenObserve] = useState(false)
|
| 76 |
-
const [screenObserveInterval, setScreenObserveInterval] = useState(60)
|
| 77 |
const [language, setLanguage] = useState<'zh' | 'en'>(() => navigator.language.startsWith('zh') ? 'zh' : 'en')
|
| 78 |
const [sttProvider, setSttProvider] = useState<'browser' | 'groq' | 'anthropic' | 'local' | 'doubao'>('browser')
|
| 79 |
const t = (zh: string, en: string) => language === 'en' ? en : zh
|
| 80 |
-
usePassThrough(!settingsOpen
|
| 81 |
|
| 82 |
// Load persisted settings on mount
|
| 83 |
useEffect(() => {
|
|
@@ -89,18 +72,12 @@ export default function App() {
|
|
| 89 |
if (s.showText !== undefined) setShowText(s.showText)
|
| 90 |
if (s.hideUI !== undefined) setHideUI(s.hideUI)
|
| 91 |
if (s.tracking) { setTracking(s.tracking); sceneRef.current?.setTrackingMode(s.tracking) }
|
| 92 |
-
if (s.volume !== undefined) { setVolume(s.volume); LipSync.getInstance().setVolume(s.volume)
|
| 93 |
if (s.uiAlign) setUiAlign(s.uiAlign)
|
| 94 |
-
if (s.hideMood !== undefined) setHideMood(s.hideMood)
|
| 95 |
-
if (s.screenObserve !== undefined) setScreenObserve(s.screenObserve)
|
| 96 |
-
if (s.screenObserveInterval !== undefined) setScreenObserveInterval(s.screenObserveInterval)
|
| 97 |
-
if (s.currentDance) setCurrentDance(s.currentDance)
|
| 98 |
-
if (s.customDancePreset) setCustomDancePreset(s.customDancePreset)
|
| 99 |
if (s.sttProvider) setSttProvider(s.sttProvider)
|
| 100 |
if (s.language) {
|
| 101 |
setLanguage(s.language)
|
| 102 |
} else {
|
| 103 |
-
// No saved language — persist the detected system language to backend
|
| 104 |
const detected = navigator.language.startsWith('zh') ? 'zh' : 'en'
|
| 105 |
fetch(`${FRIEND_API}/settings`, {
|
| 106 |
method: 'POST',
|
|
@@ -120,10 +97,6 @@ export default function App() {
|
|
| 120 |
}).catch(() => {})
|
| 121 |
}
|
| 122 |
|
| 123 |
-
useEffect(() => {
|
| 124 |
-
// open-settings Tauri event replaced by F4 keyboard shortcut (handled below)
|
| 125 |
-
}, [])
|
| 126 |
-
|
| 127 |
useEffect(() => {
|
| 128 |
bindScene(sceneRef.current)
|
| 129 |
return () => bindScene(null)
|
|
@@ -132,7 +105,6 @@ export default function App() {
|
|
| 132 |
const handleVolumeChange = useCallback((v: number) => {
|
| 133 |
setVolume(v)
|
| 134 |
LipSync.getInstance().setVolume(v)
|
| 135 |
-
sceneRef.current?.setBgmVolume(v)
|
| 136 |
saveSettings({ volume: v })
|
| 137 |
}, [])
|
| 138 |
|
|
@@ -144,22 +116,19 @@ export default function App() {
|
|
| 144 |
|
| 145 |
const handleVrmMessage: OnVrmMessage = useCallback((msg) => {
|
| 146 |
if (msg.emotion && sceneRef.current) {
|
| 147 |
-
const action = emotionActionMap[msg.emotion]
|
| 148 |
if (msg.text) {
|
| 149 |
-
// 回复消息:表情和动作同时触发(文字出现1s后由TextBubble延迟调用)
|
| 150 |
sceneRef.current.setEmotionWithReset(msg.emotion, msg.emotionDuration ?? 5000, msg.emotionIntensity)
|
| 151 |
if (action) sceneRef.current.playAction(action)
|
| 152 |
} else {
|
| 153 |
-
// 思考阶段:hold 动作,10s 后自动 reset
|
| 154 |
sceneRef.current.setEmotionWithReset(msg.emotion, msg.emotionDuration ?? 10000, msg.emotionIntensity)
|
| 155 |
if (action) sceneRef.current.playAction(action, true)
|
| 156 |
}
|
| 157 |
}
|
| 158 |
}, [])
|
| 159 |
|
| 160 |
-
// ── Idle fidget:
|
| 161 |
const lastActivityRef = useRef(Date.now())
|
| 162 |
-
// Reset idle timer whenever a VRM message arrives
|
| 163 |
const originalHandleVrmMessage = handleVrmMessage
|
| 164 |
const handleVrmMessageWithActivity: OnVrmMessage = useCallback((msg) => {
|
| 165 |
lastActivityRef.current = Date.now()
|
|
@@ -171,84 +140,52 @@ export default function App() {
|
|
| 171 |
'happy', 'sad', 'angry', 'surprised', 'think', 'awkward',
|
| 172 |
'question', 'curious', 'neutral', 'love', 'flirty', 'greeting', 'relaxed',
|
| 173 |
]
|
| 174 |
-
const
|
| 175 |
-
|
| 176 |
-
|
| 177 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 178 |
]
|
| 179 |
-
const IDLE_THRESHOLD_MS =
|
| 180 |
-
const FIDGET_CHECK_MS =
|
| 181 |
|
| 182 |
const timer = setInterval(() => {
|
| 183 |
const idleMs = Date.now() - lastActivityRef.current
|
| 184 |
if (idleMs < IDLE_THRESHOLD_MS) return
|
| 185 |
-
|
| 186 |
-
if (Math.random() > 0.5) return
|
| 187 |
-
|
| 188 |
-
const emotion = allEmotions[Math.floor(Math.random() * allEmotions.length)]
|
| 189 |
-
const action = allActions[Math.floor(Math.random() * allActions.length)]
|
| 190 |
-
const intensity = 0.4 + Math.random() * 0.4 // 0.4–0.8
|
| 191 |
-
sceneRef.current?.setEmotionWithReset(emotion, 3000 + Math.random() * 2000, intensity)
|
| 192 |
-
sceneRef.current?.playAction(action)
|
| 193 |
-
lastActivityRef.current = Date.now() // reset so we don't spam
|
| 194 |
-
}, FIDGET_CHECK_MS)
|
| 195 |
-
|
| 196 |
-
return () => clearInterval(timer)
|
| 197 |
-
}, [])
|
| 198 |
-
|
| 199 |
-
// ── Dancing mood boost: +1 every 30s ─────────
|
| 200 |
-
useEffect(() => {
|
| 201 |
-
if (!dancing) return
|
| 202 |
-
const timer = setInterval(() => {
|
| 203 |
-
fetch(`${FRIEND_API}/mood/adjust`, {
|
| 204 |
-
method: 'POST',
|
| 205 |
-
headers: { 'Content-Type': 'application/json' },
|
| 206 |
-
body: JSON.stringify({ delta: 1, max: 90 }),
|
| 207 |
-
}).catch(() => {})
|
| 208 |
-
}, 30_000)
|
| 209 |
-
return () => clearInterval(timer)
|
| 210 |
-
}, [dancing])
|
| 211 |
|
| 212 |
-
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
|
| 216 |
-
|
| 217 |
-
const doObserve = () => {
|
| 218 |
-
fetch(`${FRIEND_API}/screen/observe`, { method: 'POST' })
|
| 219 |
-
.catch(() => {})
|
| 220 |
-
}
|
| 221 |
|
| 222 |
-
|
| 223 |
-
|
| 224 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 225 |
|
| 226 |
return () => clearInterval(timer)
|
| 227 |
-
}, [screenObserve, screenObserveInterval])
|
| 228 |
-
|
| 229 |
-
// Capture VRM screenshot and upload to server (debounced to avoid duplicates)
|
| 230 |
-
const screenshotTimer = useRef<ReturnType<typeof setTimeout> | null>(null)
|
| 231 |
-
const uploadVrmScreenshot = useCallback(() => {
|
| 232 |
-
if (screenshotTimer.current) clearTimeout(screenshotTimer.current)
|
| 233 |
-
screenshotTimer.current = setTimeout(() => {
|
| 234 |
-
screenshotTimer.current = null
|
| 235 |
-
const dataUrl = sceneRef.current?.captureScreenshot()
|
| 236 |
-
if (!dataUrl) return
|
| 237 |
-
fetch(`${FRIEND_API}/persona/screenshot`, {
|
| 238 |
-
method: 'POST',
|
| 239 |
-
headers: { 'Content-Type': 'application/json' },
|
| 240 |
-
body: JSON.stringify({ image: dataUrl }),
|
| 241 |
-
}).catch(() => {})
|
| 242 |
-
}, 500)
|
| 243 |
}, [])
|
| 244 |
|
| 245 |
-
|
| 246 |
-
try {
|
| 247 |
-
await fetch(`${FRIEND_API}/context/clear`, { method: 'POST' })
|
| 248 |
-
} catch { /* ignore */ }
|
| 249 |
-
}
|
| 250 |
-
|
| 251 |
-
// 全局快捷键: Tab 展开/折叠菜单
|
| 252 |
useEffect(() => {
|
| 253 |
const onKeyDown = (e: KeyboardEvent) => {
|
| 254 |
if (e.target instanceof HTMLInputElement || e.target instanceof HTMLTextAreaElement) return
|
|
@@ -269,91 +206,7 @@ export default function App() {
|
|
| 269 |
return () => window.removeEventListener('keydown', onKeyDown)
|
| 270 |
}, [])
|
| 271 |
|
| 272 |
-
// ── Touch interaction: immediate reaction + verbal response ──────────
|
| 273 |
-
// Each region has multiple possible reactions, randomly selected (from lobe-vidol female defaults)
|
| 274 |
-
const touchReactions: Record<TouchRegion, { emotion: string; action?: string }[]> = {
|
| 275 |
-
head: [
|
| 276 |
-
{ emotion: 'relaxed', action: 'happy' },
|
| 277 |
-
{ emotion: 'relaxed', action: 'shy' },
|
| 278 |
-
{ emotion: 'relaxed', action: 'happy' },
|
| 279 |
-
{ emotion: 'relaxed', action: 'shy' },
|
| 280 |
-
{ emotion: 'angry', action: 'angryPump' },
|
| 281 |
-
{ emotion: 'relaxed', action: 'excited' },
|
| 282 |
-
],
|
| 283 |
-
arm: [
|
| 284 |
-
{ emotion: 'surprised', action: 'excited' },
|
| 285 |
-
{ emotion: 'happy', action: 'happy' },
|
| 286 |
-
{ emotion: 'relaxed', action: 'greeting' },
|
| 287 |
-
{ emotion: 'relaxed', action: 'salute' },
|
| 288 |
-
{ emotion: 'relaxed', action: 'akimbo' },
|
| 289 |
-
],
|
| 290 |
-
chest: [
|
| 291 |
-
{ emotion: 'angry', action: 'angryPump' },
|
| 292 |
-
{ emotion: 'angry', action: 'angry' },
|
| 293 |
-
{ emotion: 'angry', action: 'point' },
|
| 294 |
-
],
|
| 295 |
-
belly: [
|
| 296 |
-
{ emotion: 'angry', action: 'angryPump' },
|
| 297 |
-
{ emotion: 'angry', action: 'angry' },
|
| 298 |
-
{ emotion: 'awkward', action: 'playFingers' },
|
| 299 |
-
{ emotion: 'angry', action: 'akimbo' },
|
| 300 |
-
],
|
| 301 |
-
buttocks: [
|
| 302 |
-
{ emotion: 'angry', action: 'angryPump' },
|
| 303 |
-
{ emotion: 'angry', action: 'point' },
|
| 304 |
-
{ emotion: 'sad', action: 'shy' },
|
| 305 |
-
],
|
| 306 |
-
leg: [
|
| 307 |
-
{ emotion: 'sad', action: 'shy' },
|
| 308 |
-
{ emotion: 'angry', action: 'angry' },
|
| 309 |
-
{ emotion: 'angry', action: 'point' },
|
| 310 |
-
{ emotion: 'angry', action: 'angryPump' },
|
| 311 |
-
{ emotion: 'awkward', action: 'playFingers' },
|
| 312 |
-
],
|
| 313 |
-
}
|
| 314 |
-
|
| 315 |
-
const regionLabels: Record<TouchRegion, string> = language === 'en'
|
| 316 |
-
? { head: 'head', arm: 'arm', chest: 'chest', belly: 'belly', buttocks: 'buttocks', leg: 'leg' }
|
| 317 |
-
: { head: '头', arm: '手臂', chest: '胸', belly: '肚子', buttocks: '屁股', leg: '腿' }
|
| 318 |
-
|
| 319 |
-
const handleTouch = useCallback((region: TouchRegion) => {
|
| 320 |
-
const reactions = touchReactions[region]
|
| 321 |
-
if (!reactions?.length) return
|
| 322 |
-
const visual = reactions[Math.floor(Math.random() * reactions.length)]
|
| 323 |
-
|
| 324 |
-
// Immediate visual feedback (always)
|
| 325 |
-
lastActivityRef.current = Date.now()
|
| 326 |
-
sceneRef.current?.setEmotionWithReset(visual.emotion, 3000, 0.8)
|
| 327 |
-
if (visual.action) sceneRef.current?.playAction(visual.action)
|
| 328 |
-
|
| 329 |
-
const now = Date.now()
|
| 330 |
-
const prompt = language === 'en'
|
| 331 |
-
? `[The user touched your ${regionLabels[region]}]`
|
| 332 |
-
: `[用户摸了摸你的${regionLabels[region]}]`
|
| 333 |
-
|
| 334 |
-
// Always write to session history (3s cooldown)
|
| 335 |
-
if (now - lastTouchMemoTime > TOUCH_MEMO_COOLDOWN) {
|
| 336 |
-
lastTouchMemoTime = now
|
| 337 |
-
fetch(`${FRIEND_API}/session/memo`, {
|
| 338 |
-
method: 'POST',
|
| 339 |
-
headers: { 'Content-Type': 'application/json' },
|
| 340 |
-
body: JSON.stringify({ text: prompt }),
|
| 341 |
-
}).catch(() => {})
|
| 342 |
-
}
|
| 343 |
-
|
| 344 |
-
// Send to backend for verbal response (60s cooldown, 50% chance)
|
| 345 |
-
if (now - lastTouchChatTime > TOUCH_CHAT_COOLDOWN && Math.random() < 0.5) {
|
| 346 |
-
lastTouchChatTime = now
|
| 347 |
-
fetch(`${FRIEND_API}/touch`, {
|
| 348 |
-
method: 'POST',
|
| 349 |
-
headers: { 'Content-Type': 'application/json' },
|
| 350 |
-
body: JSON.stringify({ region, prompt }),
|
| 351 |
-
}).catch(() => {})
|
| 352 |
-
}
|
| 353 |
-
}, [])
|
| 354 |
-
|
| 355 |
const togglePin = async () => {
|
| 356 |
-
// Window pinning not available in web mode
|
| 357 |
setPinned((v) => !v)
|
| 358 |
}
|
| 359 |
|
|
@@ -368,20 +221,14 @@ export default function App() {
|
|
| 368 |
}}
|
| 369 |
>
|
| 370 |
<ResizeHandles />
|
| 371 |
-
<VRMScene ref={sceneRef} modelPath={modelPath}
|
| 372 |
-
{!hideMood && <MoodIndicator uiAlign={uiAlign} />}
|
| 373 |
<TextBubble onMessage={handleVrmMessageWithActivity} enabled={showText} ttsEnabled={ttsEnabled} />
|
| 374 |
-
{!hideUI && <ChatInput uiAlign={uiAlign}
|
| 375 |
-
<HistoryPanel
|
| 376 |
-
visible={historyOpen}
|
| 377 |
-
onClose={() => setHistoryOpen(false)}
|
| 378 |
-
language={language}
|
| 379 |
-
/>
|
| 380 |
<SettingsPanel
|
| 381 |
visible={settingsOpen}
|
| 382 |
onClose={() => setSettingsOpen(false)}
|
| 383 |
currentModel={modelPath}
|
| 384 |
-
onModelChange={(m) => { setModelPath(m);
|
| 385 |
hideUI={hideUI}
|
| 386 |
onHideUIChange={(v) => { setHideUI(v); saveSettings({ hideUI: v }) }}
|
| 387 |
showText={showText}
|
|
@@ -394,23 +241,10 @@ export default function App() {
|
|
| 394 |
onVolumeChange={handleVolumeChange}
|
| 395 |
uiAlign={uiAlign}
|
| 396 |
onUiAlignChange={(v) => { setUiAlign(v); saveSettings({ uiAlign: v }) }}
|
| 397 |
-
hideMood={hideMood}
|
| 398 |
-
onHideMoodChange={(v) => { setHideMood(v); saveSettings({ hideMood: v }) }}
|
| 399 |
-
screenObserve={screenObserve}
|
| 400 |
-
onScreenObserveChange={(v) => { setScreenObserve(v); saveSettings({ screenObserve: v }) }}
|
| 401 |
-
screenObserveInterval={screenObserveInterval}
|
| 402 |
-
onScreenObserveIntervalChange={(v) => { setScreenObserveInterval(v); saveSettings({ screenObserveInterval: v }) }}
|
| 403 |
-
captureVrmScreenshot={() => sceneRef.current?.captureScreenshot() ?? null}
|
| 404 |
language={language}
|
| 405 |
onLanguageChange={(v) => { setLanguage(v); saveSettings({ language: v }) }}
|
| 406 |
sttProvider={sttProvider}
|
| 407 |
onSttProviderChange={(v) => { setSttProvider(v); saveSettings({ sttProvider: v }) }}
|
| 408 |
-
currentDance={currentDance}
|
| 409 |
-
onDanceChange={(id, preset) => {
|
| 410 |
-
setCurrentDance(id)
|
| 411 |
-
setCustomDancePreset(preset)
|
| 412 |
-
saveSettings({ currentDance: id, customDancePreset: preset })
|
| 413 |
-
}}
|
| 414 |
/>
|
| 415 |
{!hideUI && <div
|
| 416 |
style={{
|
|
@@ -506,37 +340,6 @@ export default function App() {
|
|
| 506 |
>
|
| 507 |
<Rotate3D size={16} />
|
| 508 |
</button>
|
| 509 |
-
<button
|
| 510 |
-
onClick={() => {
|
| 511 |
-
if (dancing) {
|
| 512 |
-
sceneRef.current?.reset()
|
| 513 |
-
setDancing(false)
|
| 514 |
-
} else {
|
| 515 |
-
const label = currentDance.startsWith('custom:') && customDancePreset
|
| 516 |
-
? customDancePreset.label
|
| 517 |
-
: dancePresets[currentDance]?.label ?? currentDance
|
| 518 |
-
if (currentDance.startsWith('custom:') && customDancePreset) {
|
| 519 |
-
sceneRef.current?.playDance(customDancePreset)
|
| 520 |
-
} else {
|
| 521 |
-
sceneRef.current?.playDance(currentDance)
|
| 522 |
-
}
|
| 523 |
-
setDancing(true)
|
| 524 |
-
// Record dance event in session history (no LLM reply)
|
| 525 |
-
fetch(`${FRIEND_API}/session/memo`, {
|
| 526 |
-
method: 'POST',
|
| 527 |
-
headers: { 'Content-Type': 'application/json' },
|
| 528 |
-
body: JSON.stringify({ text: `[用户邀请你跳了一支舞:${label}]` }),
|
| 529 |
-
}).catch(() => {})
|
| 530 |
-
}
|
| 531 |
-
}}
|
| 532 |
-
style={{
|
| 533 |
-
...btnStyle,
|
| 534 |
-
...(dancing ? { background: 'rgba(34, 197, 94, 0.6)' } : {}),
|
| 535 |
-
}}
|
| 536 |
-
title={dancing ? t('停止跳舞', 'Stop Dance') : t('跳舞', 'Dance')}
|
| 537 |
-
>
|
| 538 |
-
<Music size={16} />
|
| 539 |
-
</button>
|
| 540 |
</>}
|
| 541 |
</div>}
|
| 542 |
</div>
|
|
|
|
| 1 |
import { useEffect, useRef, useState, useCallback } from 'react'
|
| 2 |
import { VRMScene } from './components/VRMScene'
|
| 3 |
+
import type { VRMSceneHandle } from './components/VRMScene'
|
| 4 |
import { TextBubble } from './components/TextBubble'
|
| 5 |
import type { OnVrmMessage } from './components/TextBubble'
|
| 6 |
import { ChatInput } from './components/ChatInput'
|
| 7 |
import { ResizeHandles } from './components/ResizeHandles'
|
| 8 |
import { SettingsPanel } from './components/SettingsPanel'
|
| 9 |
import { usePassThrough } from './hooks/usePassThrough'
|
|
|
|
| 10 |
import { LipSync } from './lip-sync'
|
| 11 |
import { FRIEND_API, bindScene } from './api'
|
| 12 |
+
import { Menu, Move, Rotate3D, EyeOff, Settings, RefreshCw, Pin } from 'lucide-react'
|
|
|
|
|
|
|
| 13 |
|
| 14 |
const DEFAULT_MODEL = '/friend/model1.vrm'
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15 |
|
| 16 |
+
// 情绪 → 动作映射
|
| 17 |
const emotionActionMap: Record<string, string> = {
|
| 18 |
think: 'scratchHead',
|
| 19 |
question: 'point',
|
|
|
|
| 30 |
neutral: 'salute',
|
| 31 |
}
|
| 32 |
|
|
|
|
| 33 |
const btnStyle: React.CSSProperties = {
|
| 34 |
width: 32,
|
| 35 |
height: 32,
|
|
|
|
| 51 |
const [tracking, setTracking] = useState<'mouse' | 'camera'>('mouse')
|
| 52 |
const [showText, setShowText] = useState(true)
|
| 53 |
const [collapsed, setCollapsed] = useState(false)
|
|
|
|
| 54 |
const [ttsEnabled, setTtsEnabled] = useState(true)
|
| 55 |
const [modelPath, setModelPath] = useState(DEFAULT_MODEL)
|
| 56 |
const [settingsOpen, setSettingsOpen] = useState(false)
|
|
|
|
| 57 |
const [hideUI, setHideUI] = useState(false)
|
| 58 |
const [volume, setVolume] = useState(0.5)
|
| 59 |
const [uiAlign, setUiAlign] = useState<'left' | 'right'>('right')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 60 |
const [language, setLanguage] = useState<'zh' | 'en'>(() => navigator.language.startsWith('zh') ? 'zh' : 'en')
|
| 61 |
const [sttProvider, setSttProvider] = useState<'browser' | 'groq' | 'anthropic' | 'local' | 'doubao'>('browser')
|
| 62 |
const t = (zh: string, en: string) => language === 'en' ? en : zh
|
| 63 |
+
usePassThrough(!settingsOpen)
|
| 64 |
|
| 65 |
// Load persisted settings on mount
|
| 66 |
useEffect(() => {
|
|
|
|
| 72 |
if (s.showText !== undefined) setShowText(s.showText)
|
| 73 |
if (s.hideUI !== undefined) setHideUI(s.hideUI)
|
| 74 |
if (s.tracking) { setTracking(s.tracking); sceneRef.current?.setTrackingMode(s.tracking) }
|
| 75 |
+
if (s.volume !== undefined) { setVolume(s.volume); LipSync.getInstance().setVolume(s.volume) }
|
| 76 |
if (s.uiAlign) setUiAlign(s.uiAlign)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 77 |
if (s.sttProvider) setSttProvider(s.sttProvider)
|
| 78 |
if (s.language) {
|
| 79 |
setLanguage(s.language)
|
| 80 |
} else {
|
|
|
|
| 81 |
const detected = navigator.language.startsWith('zh') ? 'zh' : 'en'
|
| 82 |
fetch(`${FRIEND_API}/settings`, {
|
| 83 |
method: 'POST',
|
|
|
|
| 97 |
}).catch(() => {})
|
| 98 |
}
|
| 99 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 100 |
useEffect(() => {
|
| 101 |
bindScene(sceneRef.current)
|
| 102 |
return () => bindScene(null)
|
|
|
|
| 105 |
const handleVolumeChange = useCallback((v: number) => {
|
| 106 |
setVolume(v)
|
| 107 |
LipSync.getInstance().setVolume(v)
|
|
|
|
| 108 |
saveSettings({ volume: v })
|
| 109 |
}, [])
|
| 110 |
|
|
|
|
| 116 |
|
| 117 |
const handleVrmMessage: OnVrmMessage = useCallback((msg) => {
|
| 118 |
if (msg.emotion && sceneRef.current) {
|
| 119 |
+
const action = msg.action || emotionActionMap[msg.emotion]
|
| 120 |
if (msg.text) {
|
|
|
|
| 121 |
sceneRef.current.setEmotionWithReset(msg.emotion, msg.emotionDuration ?? 5000, msg.emotionIntensity)
|
| 122 |
if (action) sceneRef.current.playAction(action)
|
| 123 |
} else {
|
|
|
|
| 124 |
sceneRef.current.setEmotionWithReset(msg.emotion, msg.emotionDuration ?? 10000, msg.emotionIntensity)
|
| 125 |
if (action) sceneRef.current.playAction(action, true)
|
| 126 |
}
|
| 127 |
}
|
| 128 |
}, [])
|
| 129 |
|
| 130 |
+
// ── Idle fidget: natural micro-movements when no activity ─────────────────
|
| 131 |
const lastActivityRef = useRef(Date.now())
|
|
|
|
| 132 |
const originalHandleVrmMessage = handleVrmMessage
|
| 133 |
const handleVrmMessageWithActivity: OnVrmMessage = useCallback((msg) => {
|
| 134 |
lastActivityRef.current = Date.now()
|
|
|
|
| 140 |
'happy', 'sad', 'angry', 'surprised', 'think', 'awkward',
|
| 141 |
'question', 'curious', 'neutral', 'love', 'flirty', 'greeting', 'relaxed',
|
| 142 |
]
|
| 143 |
+
const idleTimers: Array<{ emotion?: string; action?: string; minIdle: number }> = [
|
| 144 |
+
// Emotion-only (subtle facial changes, no body action)
|
| 145 |
+
{ emotion: 'think', minIdle: 8 },
|
| 146 |
+
{ emotion: 'curious', minIdle: 12 },
|
| 147 |
+
{ emotion: 'relaxed', minIdle: 15 },
|
| 148 |
+
{ emotion: 'happy', minIdle: 20 },
|
| 149 |
+
// Small body actions (no strong emotion)
|
| 150 |
+
{ action: 'stretch', minIdle: 25 },
|
| 151 |
+
{ action: 'scratchHead', minIdle: 10 },
|
| 152 |
+
{ action: 'playFingers', minIdle: 15 },
|
| 153 |
+
{ action: 'akimbo', minIdle: 30 },
|
| 154 |
+
// Combined emotion + action
|
| 155 |
+
{ emotion: 'think', action: 'scratchHead', minIdle: 18 },
|
| 156 |
+
{ emotion: 'relaxed', action: 'akimbo', minIdle: 35 },
|
| 157 |
+
{ emotion: 'curious', action: 'point', minIdle: 22 },
|
| 158 |
+
{ emotion: 'happy', action: 'shy', minIdle: 28 },
|
| 159 |
+
{ emotion: 'surprised', action: 'excited', minIdle: 40 },
|
| 160 |
]
|
| 161 |
+
const IDLE_THRESHOLD_MS = 15_000
|
| 162 |
+
const FIDGET_CHECK_MS = 8_000
|
| 163 |
|
| 164 |
const timer = setInterval(() => {
|
| 165 |
const idleMs = Date.now() - lastActivityRef.current
|
| 166 |
if (idleMs < IDLE_THRESHOLD_MS) return
|
| 167 |
+
if (Math.random() > 0.55) return
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 168 |
|
| 169 |
+
// Pick a weighted-random fidget based on how long we've been idle
|
| 170 |
+
const candidates = idleTimers.filter((t) => idleMs >= t.minIdle * 1000)
|
| 171 |
+
if (candidates.length === 0) return
|
| 172 |
+
const pick = candidates[Math.floor(Math.random() * candidates.length)]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 173 |
|
| 174 |
+
if (pick.emotion) {
|
| 175 |
+
const intensity = 0.3 + Math.random() * 0.5
|
| 176 |
+
const duration = 2000 + Math.random() * 3000
|
| 177 |
+
sceneRef.current?.setEmotionWithReset(pick.emotion, duration, intensity)
|
| 178 |
+
}
|
| 179 |
+
if (pick.action) {
|
| 180 |
+
sceneRef.current?.playAction(pick.action)
|
| 181 |
+
}
|
| 182 |
+
lastActivityRef.current = Date.now()
|
| 183 |
+
}, FIDGET_CHECK_MS)
|
| 184 |
|
| 185 |
return () => clearInterval(timer)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 186 |
}, [])
|
| 187 |
|
| 188 |
+
// 全局快捷键
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 189 |
useEffect(() => {
|
| 190 |
const onKeyDown = (e: KeyboardEvent) => {
|
| 191 |
if (e.target instanceof HTMLInputElement || e.target instanceof HTMLTextAreaElement) return
|
|
|
|
| 206 |
return () => window.removeEventListener('keydown', onKeyDown)
|
| 207 |
}, [])
|
| 208 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 209 |
const togglePin = async () => {
|
|
|
|
| 210 |
setPinned((v) => !v)
|
| 211 |
}
|
| 212 |
|
|
|
|
| 221 |
}}
|
| 222 |
>
|
| 223 |
<ResizeHandles />
|
| 224 |
+
<VRMScene ref={sceneRef} modelPath={modelPath} />
|
|
|
|
| 225 |
<TextBubble onMessage={handleVrmMessageWithActivity} enabled={showText} ttsEnabled={ttsEnabled} />
|
| 226 |
+
{!hideUI && <ChatInput uiAlign={uiAlign} language={language} sttProvider={sttProvider} />}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 227 |
<SettingsPanel
|
| 228 |
visible={settingsOpen}
|
| 229 |
onClose={() => setSettingsOpen(false)}
|
| 230 |
currentModel={modelPath}
|
| 231 |
+
onModelChange={(m) => { setModelPath(m); saveSettings({ modelPath: m }) }}
|
| 232 |
hideUI={hideUI}
|
| 233 |
onHideUIChange={(v) => { setHideUI(v); saveSettings({ hideUI: v }) }}
|
| 234 |
showText={showText}
|
|
|
|
| 241 |
onVolumeChange={handleVolumeChange}
|
| 242 |
uiAlign={uiAlign}
|
| 243 |
onUiAlignChange={(v) => { setUiAlign(v); saveSettings({ uiAlign: v }) }}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 244 |
language={language}
|
| 245 |
onLanguageChange={(v) => { setLanguage(v); saveSettings({ language: v }) }}
|
| 246 |
sttProvider={sttProvider}
|
| 247 |
onSttProviderChange={(v) => { setSttProvider(v); saveSettings({ sttProvider: v }) }}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 248 |
/>
|
| 249 |
{!hideUI && <div
|
| 250 |
style={{
|
|
|
|
| 340 |
>
|
| 341 |
<Rotate3D size={16} />
|
| 342 |
</button>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 343 |
</>}
|
| 344 |
</div>}
|
| 345 |
</div>
|
src/components/friend/frontend/components/ChatInput.tsx
CHANGED
|
@@ -1,198 +1,72 @@
|
|
| 1 |
import { useState, useRef, useCallback, useEffect } from 'react'
|
| 2 |
-
import {
|
| 3 |
import { FRIEND_API } from '../api'
|
| 4 |
import { useServerStt, type SttProvider } from '../hooks/useServerStt'
|
| 5 |
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
export function ChatInput({ visible = true, onActiveChange, uiAlign = 'right', onHistoryOpen, onNewSession, language = 'zh', sttProvider = 'browser' }: { visible?: boolean; onActiveChange?: (hasText: boolean) => void; uiAlign?: 'left' | 'right'; onHistoryOpen?: () => void; onNewSession?: () => void; language?: 'zh' | 'en'; sttProvider?: SttProvider }) {
|
| 9 |
const t = (zh: string, en: string) => language === 'en' ? en : zh
|
| 10 |
-
const [
|
| 11 |
const [text, setText] = useState('')
|
| 12 |
-
const [sending, setSending] = useState(false)
|
| 13 |
const [recording, setRecording] = useState(false)
|
| 14 |
-
const [closing, setClosing] = useState(false)
|
| 15 |
-
const [menuOpen, setMenuOpen] = useState(false)
|
| 16 |
-
const [voiceCallActive, setVoiceCallActive] = useState(false)
|
| 17 |
-
const inputRef = useRef<HTMLInputElement>(null)
|
| 18 |
const voiceCallActiveRef = useRef(false)
|
| 19 |
-
const endVoiceCallRef = useRef<() => void>(() => {})
|
| 20 |
const serverStt = useServerStt()
|
| 21 |
-
|
| 22 |
-
const closeBar = useCallback(() => {
|
| 23 |
-
if (closing) return
|
| 24 |
-
setMenuOpen(false)
|
| 25 |
-
setClosing(true)
|
| 26 |
-
setTimeout(() => { setOpen(false); setClosing(false) }, 220)
|
| 27 |
-
}, [closing])
|
| 28 |
-
|
| 29 |
-
const send = useCallback(async (msg?: string) => {
|
| 30 |
-
const finalMsg = (msg ?? text).trim()
|
| 31 |
-
if (!finalMsg || sending) return
|
| 32 |
-
setSending(true)
|
| 33 |
-
setText('')
|
| 34 |
-
onActiveChange?.(false)
|
| 35 |
-
try {
|
| 36 |
-
await fetch(`${FRIEND_API}/chat`, {
|
| 37 |
-
method: 'POST',
|
| 38 |
-
headers: { 'Content-Type': 'application/json' },
|
| 39 |
-
body: JSON.stringify({ message: finalMsg }),
|
| 40 |
-
})
|
| 41 |
-
} catch (err) {
|
| 42 |
-
console.error('Failed to send message:', err)
|
| 43 |
-
} finally {
|
| 44 |
-
setSending(false)
|
| 45 |
-
}
|
| 46 |
-
}, [text, sending])
|
| 47 |
-
|
| 48 |
-
const startRecording = useCallback(async () => {
|
| 49 |
-
// All voice capture is server-side via cpal (no getUserMedia needed)
|
| 50 |
-
try {
|
| 51 |
-
await serverStt.startPushToTalk(sttProvider, language === 'en' ? 'en' : 'zh')
|
| 52 |
-
setRecording(true)
|
| 53 |
-
setOpen(true)
|
| 54 |
-
} catch (err) {
|
| 55 |
-
console.error('Server STT start error:', err)
|
| 56 |
-
setRecording(false)
|
| 57 |
-
}
|
| 58 |
-
}, [sttProvider, serverStt, language])
|
| 59 |
-
|
| 60 |
-
const stopRecording = useCallback(async () => {
|
| 61 |
-
if (sttProvider === 'browser') {
|
| 62 |
-
// Browser STT may not be available, just stop
|
| 63 |
-
setRecording(false)
|
| 64 |
-
return
|
| 65 |
-
}
|
| 66 |
-
setRecording(false)
|
| 67 |
-
try {
|
| 68 |
-
const text = await serverStt.stopPushToTalk()
|
| 69 |
-
setText(text)
|
| 70 |
-
onActiveChange?.(text.length > 0)
|
| 71 |
-
} catch (err) {
|
| 72 |
-
console.error('Server STT stop error:', err)
|
| 73 |
-
}
|
| 74 |
-
}, [serverStt, sttProvider, onActiveChange])
|
| 75 |
-
|
| 76 |
-
const handleMouseLeave = useCallback(() => {
|
| 77 |
-
if (recording) stopRecording()
|
| 78 |
-
}, [recording, stopRecording])
|
| 79 |
-
|
| 80 |
-
// --- Voice Call: delayed TTS interrupt ---
|
| 81 |
-
const interruptTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null)
|
| 82 |
-
const scheduleInterrupt = useCallback(() => {
|
| 83 |
-
if (interruptTimerRef.current) return
|
| 84 |
-
interruptTimerRef.current = setTimeout(() => {
|
| 85 |
-
interruptTimerRef.current = null
|
| 86 |
-
;(window as any).__clawInterruptAudio?.()
|
| 87 |
-
}, 1000)
|
| 88 |
-
}, [])
|
| 89 |
-
const cancelInterrupt = useCallback(() => {
|
| 90 |
-
if (interruptTimerRef.current) { clearTimeout(interruptTimerRef.current); interruptTimerRef.current = null }
|
| 91 |
-
}, [])
|
| 92 |
-
|
| 93 |
-
// Voice call display timer — clear text after 3s
|
| 94 |
const displayTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null)
|
| 95 |
|
| 96 |
-
// --- Voice Call
|
| 97 |
const startVoiceCall = useCallback(async () => {
|
| 98 |
setVoiceCallActive(true)
|
| 99 |
voiceCallActiveRef.current = true
|
| 100 |
setText('')
|
| 101 |
-
setOpen(true)
|
| 102 |
|
| 103 |
serverStt.startStreaming(
|
| 104 |
sttProvider,
|
| 105 |
-
(
|
| 106 |
-
|
| 107 |
-
|
| 108 |
-
// We only need to display the transcript in the input bar briefly.
|
| 109 |
-
if (text.trim()) {
|
| 110 |
-
setText(text)
|
| 111 |
-
onActiveChange?.(true)
|
| 112 |
-
// Auto-clear after 3s
|
| 113 |
if (displayTimerRef.current) clearTimeout(displayTimerRef.current)
|
| 114 |
displayTimerRef.current = setTimeout(() => {
|
| 115 |
if (voiceCallActiveRef.current) {
|
| 116 |
setText('')
|
| 117 |
-
onActiveChange?.(false)
|
| 118 |
}
|
| 119 |
}, 3000)
|
| 120 |
}
|
| 121 |
},
|
| 122 |
(err) => {
|
| 123 |
-
console.error('
|
| 124 |
if (voiceCallActiveRef.current) {
|
| 125 |
-
setText(`
|
| 126 |
setTimeout(() => {
|
| 127 |
setText('')
|
| 128 |
-
|
| 129 |
}, 3000)
|
| 130 |
}
|
| 131 |
},
|
| 132 |
language === 'en' ? 'en' : 'zh',
|
| 133 |
)
|
| 134 |
setRecording(true)
|
| 135 |
-
}, [sttProvider, serverStt, language
|
| 136 |
|
| 137 |
-
// --- Voice Call: end ---
|
| 138 |
const endVoiceCall = useCallback(async () => {
|
| 139 |
-
if (!voiceCallActiveRef.current) return
|
| 140 |
voiceCallActiveRef.current = false
|
| 141 |
setVoiceCallActive(false)
|
| 142 |
setRecording(false)
|
| 143 |
-
|
| 144 |
-
// Stop VAD and clean up
|
| 145 |
const finalText = await serverStt.stopStreaming()
|
| 146 |
-
|
| 147 |
-
// Send any remaining transcript (no-op if empty — segments were already
|
| 148 |
-
// sent to CLI by transcribeAudioSegment() during the call).
|
| 149 |
if (finalText.trim()) {
|
| 150 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 151 |
}
|
| 152 |
-
|
| 153 |
-
cancelInterrupt()
|
| 154 |
-
|
| 155 |
setText('')
|
| 156 |
-
|
| 157 |
-
onActiveChange?.(false)
|
| 158 |
-
}, [onActiveChange, cancelInterrupt, serverStt, send])
|
| 159 |
-
|
| 160 |
-
endVoiceCallRef.current = endVoiceCall
|
| 161 |
-
|
| 162 |
-
const handleKeyDown = (e: React.KeyboardEvent) => {
|
| 163 |
-
if (e.key === 'Enter' && !e.shiftKey) {
|
| 164 |
-
e.preventDefault()
|
| 165 |
-
send()
|
| 166 |
-
}
|
| 167 |
-
if (e.key === 'Escape') {
|
| 168 |
-
closeBar()
|
| 169 |
-
}
|
| 170 |
-
if ((e.key === 'Delete') || (e.key === 'd' && e.ctrlKey)) {
|
| 171 |
-
e.preventDefault()
|
| 172 |
-
setText('')
|
| 173 |
-
onActiveChange?.(false)
|
| 174 |
-
}
|
| 175 |
-
}
|
| 176 |
|
| 177 |
// Global keyboard shortcuts
|
| 178 |
useEffect(() => {
|
| 179 |
const onGlobalKeyDown = (e: KeyboardEvent) => {
|
| 180 |
-
if (
|
| 181 |
-
const inInput = e.target instanceof HTMLInputElement || e.target instanceof HTMLTextAreaElement
|
| 182 |
-
|
| 183 |
-
if (e.key === 'Enter' && !e.shiftKey && !inInput) {
|
| 184 |
-
e.preventDefault()
|
| 185 |
-
if (!open) {
|
| 186 |
-
setMenuOpen(false)
|
| 187 |
-
setOpen(true)
|
| 188 |
-
}
|
| 189 |
-
setTimeout(() => inputRef.current?.focus(), 50)
|
| 190 |
-
}
|
| 191 |
-
|
| 192 |
-
if (e.key === 'Escape' && open && !inInput) {
|
| 193 |
-
e.preventDefault()
|
| 194 |
-
closeBar()
|
| 195 |
-
}
|
| 196 |
|
| 197 |
if (e.key === 'F2') {
|
| 198 |
e.preventDefault()
|
|
@@ -205,58 +79,25 @@ export function ChatInput({ visible = true, onActiveChange, uiAlign = 'right', o
|
|
| 205 |
}
|
| 206 |
window.addEventListener('keydown', onGlobalKeyDown)
|
| 207 |
return () => window.removeEventListener('keydown', onGlobalKeyDown)
|
| 208 |
-
}, [
|
| 209 |
|
| 210 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 211 |
|
| 212 |
-
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
|
| 216 |
-
|
| 217 |
-
|
| 218 |
-
title={t('语音通话 (F2)', 'Voice Call (F2)')}
|
| 219 |
-
>
|
| 220 |
-
<Phone size={16} />
|
| 221 |
-
</button>
|
| 222 |
-
<button
|
| 223 |
-
onClick={() => {
|
| 224 |
-
setMenuOpen(false)
|
| 225 |
-
setOpen(true)
|
| 226 |
-
setTimeout(() => inputRef.current?.focus(), 50)
|
| 227 |
-
}}
|
| 228 |
-
style={fabStyle}
|
| 229 |
-
title={t('发送消息 (Enter)', 'Send Message (Enter)')}
|
| 230 |
-
>
|
| 231 |
-
<MessageCircle size={16} />
|
| 232 |
-
</button>
|
| 233 |
-
</div>
|
| 234 |
-
)
|
| 235 |
-
}
|
| 236 |
|
| 237 |
if (voiceCallActive) {
|
| 238 |
return (
|
| 239 |
<div style={barStyle}>
|
| 240 |
<div style={{ flex: 1, position: 'relative' }}>
|
| 241 |
-
<button
|
| 242 |
-
onClick={() => setMenuOpen((v) => !v)}
|
| 243 |
-
style={{ ...inlineBtnLeft, color: menuOpen ? 'rgba(100, 160, 255, 0.9)' : 'rgba(255, 255, 255, 0.45)' }}
|
| 244 |
-
title={t('更多', 'More')}
|
| 245 |
-
>
|
| 246 |
-
<Plus size={22} />
|
| 247 |
-
</button>
|
| 248 |
-
{menuOpen && (
|
| 249 |
-
<div style={popupMenuStyle}>
|
| 250 |
-
<button onClick={() => { setMenuOpen(false); onHistoryOpen?.() }} style={popupItemStyle}>
|
| 251 |
-
<History size={14} />
|
| 252 |
-
<span>{t('对话历史', 'History')}</span>
|
| 253 |
-
</button>
|
| 254 |
-
<button onClick={() => { setMenuOpen(false); onNewSession?.() }} style={popupItemStyle}>
|
| 255 |
-
<SquarePen size={14} />
|
| 256 |
-
<span>{t('新会话', 'New Chat')}</span>
|
| 257 |
-
</button>
|
| 258 |
-
</div>
|
| 259 |
-
)}
|
| 260 |
<div
|
| 261 |
style={{
|
| 262 |
...inputStyle,
|
|
@@ -274,8 +115,15 @@ export function ChatInput({ visible = true, onActiveChange, uiAlign = 'right', o
|
|
| 274 |
</div>
|
| 275 |
<span
|
| 276 |
style={{
|
| 277 |
-
|
|
|
|
|
|
|
| 278 |
right: 44,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 279 |
color: recording ? 'rgba(255, 80, 80, 0.9)' : 'rgba(80, 200, 120, 0.9)',
|
| 280 |
animation: recording ? 'claw-pulse 0.8s ease-in-out infinite' : 'none',
|
| 281 |
transition: 'color 0.2s',
|
|
@@ -286,7 +134,23 @@ export function ChatInput({ visible = true, onActiveChange, uiAlign = 'right', o
|
|
| 286 |
</span>
|
| 287 |
<button
|
| 288 |
onClick={endVoiceCall}
|
| 289 |
-
style={{
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 290 |
title={t('挂断 (F2)', 'Hang up (F2)')}
|
| 291 |
>
|
| 292 |
<PhoneOff size={22} />
|
|
@@ -297,89 +161,33 @@ export function ChatInput({ visible = true, onActiveChange, uiAlign = 'right', o
|
|
| 297 |
}
|
| 298 |
|
| 299 |
return (
|
| 300 |
-
<>
|
| 301 |
-
|
| 302 |
-
|
| 303 |
-
|
| 304 |
-
|
| 305 |
-
|
| 306 |
-
|
| 307 |
-
:
|
| 308 |
-
|
| 309 |
-
|
| 310 |
-
|
| 311 |
-
|
| 312 |
-
|
| 313 |
-
|
| 314 |
-
|
| 315 |
-
|
| 316 |
-
|
| 317 |
-
|
| 318 |
-
|
| 319 |
-
|
| 320 |
-
|
| 321 |
-
|
| 322 |
-
|
| 323 |
-
</button>
|
| 324 |
-
<button onClick={() => { setMenuOpen(false); onNewSession?.() }} style={popupItemStyle}>
|
| 325 |
-
<SquarePen size={14} />
|
| 326 |
-
<span>{t('新会话', 'New Chat')}</span>
|
| 327 |
-
</button>
|
| 328 |
-
</div>
|
| 329 |
-
)}
|
| 330 |
-
<input
|
| 331 |
-
ref={inputRef}
|
| 332 |
-
value={text}
|
| 333 |
-
onChange={(e) => {
|
| 334 |
-
setText(e.target.value)
|
| 335 |
-
onActiveChange?.(e.target.value.length > 0)
|
| 336 |
-
}}
|
| 337 |
-
onKeyDown={handleKeyDown}
|
| 338 |
-
placeholder={sending ? t('思考中...', 'Thinking...') : ''}
|
| 339 |
-
disabled={sending}
|
| 340 |
-
style={{ ...inputStyle, paddingLeft: 48, paddingRight: 80 }}
|
| 341 |
-
autoFocus
|
| 342 |
-
/>
|
| 343 |
-
<button
|
| 344 |
-
onMouseDown={startRecording}
|
| 345 |
-
onMouseUp={stopRecording}
|
| 346 |
-
onMouseLeave={handleMouseLeave}
|
| 347 |
-
style={{ ...inlineBtnRight, right: 44, color: recording ? 'rgba(255, 80, 80, 0.9)' : 'rgba(255, 255, 255, 0.45)' }}
|
| 348 |
-
title={t('按住说话', 'Hold to speak')}
|
| 349 |
-
>
|
| 350 |
-
<Mic size={22} />
|
| 351 |
-
</button>
|
| 352 |
-
<button
|
| 353 |
-
onClick={() => text.trim() ? send() : closeBar()}
|
| 354 |
-
disabled={sending}
|
| 355 |
-
style={{ ...inlineBtnRight, right: 12, color: text.trim() ? 'rgba(100, 160, 255, 0.9)' : 'rgba(255, 255, 255, 0.45)' }}
|
| 356 |
-
title={text.trim() ? t('发送 (Enter)', 'Send (Enter)') : t('收起 (Esc)', 'Collapse (Esc)')}
|
| 357 |
-
>
|
| 358 |
-
{sending ? <Loader size={22} style={{ animation: 'spin 1s linear infinite' }} /> : text.trim() ? <Send size={22} /> : <ChevronDown size={22} />}
|
| 359 |
-
</button>
|
| 360 |
-
</div>
|
| 361 |
</div>
|
| 362 |
-
</>
|
| 363 |
)
|
| 364 |
}
|
| 365 |
|
| 366 |
-
const fabStyle: React.CSSProperties = {
|
| 367 |
-
width: 32,
|
| 368 |
-
height: 32,
|
| 369 |
-
border: 'none',
|
| 370 |
-
borderRadius: 6,
|
| 371 |
-
background: 'rgba(125, 125, 125, 0.28)',
|
| 372 |
-
backdropFilter: 'blur(6px)',
|
| 373 |
-
color: 'rgba(255, 255, 255, 0.8)',
|
| 374 |
-
fontSize: 16,
|
| 375 |
-
cursor: 'pointer',
|
| 376 |
-
display: 'flex',
|
| 377 |
-
alignItems: 'center',
|
| 378 |
-
justifyContent: 'center',
|
| 379 |
-
zIndex: 300,
|
| 380 |
-
pointerEvents: 'auto',
|
| 381 |
-
}
|
| 382 |
-
|
| 383 |
const barStyle: React.CSSProperties = {
|
| 384 |
position: 'absolute',
|
| 385 |
bottom: 16,
|
|
@@ -394,7 +202,7 @@ const barStyle: React.CSSProperties = {
|
|
| 394 |
const inputStyle: React.CSSProperties = {
|
| 395 |
width: '100%',
|
| 396 |
height: 50,
|
| 397 |
-
boxSizing: 'border-box'
|
| 398 |
border: '1px solid rgba(255, 255, 255, 0.2)',
|
| 399 |
borderRadius: 25,
|
| 400 |
background: 'rgba(0, 0, 0, 0.4)',
|
|
@@ -405,70 +213,3 @@ const inputStyle: React.CSSProperties = {
|
|
| 405 |
outline: 'none',
|
| 406 |
fontFamily: '"Segoe UI", "Microsoft YaHei", sans-serif',
|
| 407 |
}
|
| 408 |
-
|
| 409 |
-
const inlineBtnLeft: React.CSSProperties = {
|
| 410 |
-
position: 'absolute',
|
| 411 |
-
left: 12,
|
| 412 |
-
top: '50%',
|
| 413 |
-
transform: 'translateY(-50%)',
|
| 414 |
-
width: 32,
|
| 415 |
-
height: 32,
|
| 416 |
-
border: 'none',
|
| 417 |
-
borderRadius: 4,
|
| 418 |
-
background: 'transparent',
|
| 419 |
-
cursor: 'pointer',
|
| 420 |
-
display: 'flex',
|
| 421 |
-
alignItems: 'center',
|
| 422 |
-
justifyContent: 'center',
|
| 423 |
-
padding: 0,
|
| 424 |
-
zIndex: 2,
|
| 425 |
-
}
|
| 426 |
-
|
| 427 |
-
const inlineBtnRight: React.CSSProperties = {
|
| 428 |
-
position: 'absolute',
|
| 429 |
-
top: '50%',
|
| 430 |
-
transform: 'translateY(-50%)',
|
| 431 |
-
width: 32,
|
| 432 |
-
height: 32,
|
| 433 |
-
border: 'none',
|
| 434 |
-
borderRadius: 4,
|
| 435 |
-
background: 'transparent',
|
| 436 |
-
cursor: 'pointer',
|
| 437 |
-
display: 'flex',
|
| 438 |
-
alignItems: 'center',
|
| 439 |
-
justifyContent: 'center',
|
| 440 |
-
padding: 0,
|
| 441 |
-
zIndex: 1,
|
| 442 |
-
}
|
| 443 |
-
|
| 444 |
-
const popupMenuStyle: React.CSSProperties = {
|
| 445 |
-
position: 'absolute',
|
| 446 |
-
left: 4,
|
| 447 |
-
bottom: '100%',
|
| 448 |
-
marginBottom: 6,
|
| 449 |
-
background: 'rgba(30, 30, 40, 0.95)',
|
| 450 |
-
backdropFilter: 'blur(12px)',
|
| 451 |
-
borderRadius: 10,
|
| 452 |
-
border: '1px solid rgba(255, 255, 255, 0.15)',
|
| 453 |
-
boxShadow: '0 4px 16px rgba(0, 0, 0, 0.4)',
|
| 454 |
-
padding: 4,
|
| 455 |
-
display: 'flex',
|
| 456 |
-
flexDirection: 'column',
|
| 457 |
-
gap: 2,
|
| 458 |
-
zIndex: 1000,
|
| 459 |
-
}
|
| 460 |
-
|
| 461 |
-
const popupItemStyle: React.CSSProperties = {
|
| 462 |
-
display: 'flex',
|
| 463 |
-
alignItems: 'center',
|
| 464 |
-
gap: 8,
|
| 465 |
-
padding: '8px 14px',
|
| 466 |
-
border: 'none',
|
| 467 |
-
borderRadius: 8,
|
| 468 |
-
background: 'transparent',
|
| 469 |
-
color: 'rgba(255, 255, 255, 0.8)',
|
| 470 |
-
fontSize: 13,
|
| 471 |
-
cursor: 'pointer',
|
| 472 |
-
whiteSpace: 'nowrap',
|
| 473 |
-
fontFamily: '"Segoe UI", "Microsoft YaHei", sans-serif',
|
| 474 |
-
}
|
|
|
|
| 1 |
import { useState, useRef, useCallback, useEffect } from 'react'
|
| 2 |
+
import { Phone, PhoneOff, Mic } from 'lucide-react'
|
| 3 |
import { FRIEND_API } from '../api'
|
| 4 |
import { useServerStt, type SttProvider } from '../hooks/useServerStt'
|
| 5 |
|
| 6 |
+
export function ChatInput({ uiAlign = 'right', language = 'zh', sttProvider = 'browser' }: { uiAlign?: 'left' | 'right'; language?: 'zh' | 'en'; sttProvider?: SttProvider }) {
|
|
|
|
|
|
|
| 7 |
const t = (zh: string, en: string) => language === 'en' ? en : zh
|
| 8 |
+
const [voiceCallActive, setVoiceCallActive] = useState(false)
|
| 9 |
const [text, setText] = useState('')
|
|
|
|
| 10 |
const [recording, setRecording] = useState(false)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11 |
const voiceCallActiveRef = useRef(false)
|
|
|
|
| 12 |
const serverStt = useServerStt()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
const displayTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null)
|
| 14 |
|
| 15 |
+
// --- Voice Call ---
|
| 16 |
const startVoiceCall = useCallback(async () => {
|
| 17 |
setVoiceCallActive(true)
|
| 18 |
voiceCallActiveRef.current = true
|
| 19 |
setText('')
|
|
|
|
| 20 |
|
| 21 |
serverStt.startStreaming(
|
| 22 |
sttProvider,
|
| 23 |
+
(partialText, _isFinal) => {
|
| 24 |
+
if (partialText.trim()) {
|
| 25 |
+
setText(partialText)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 26 |
if (displayTimerRef.current) clearTimeout(displayTimerRef.current)
|
| 27 |
displayTimerRef.current = setTimeout(() => {
|
| 28 |
if (voiceCallActiveRef.current) {
|
| 29 |
setText('')
|
|
|
|
| 30 |
}
|
| 31 |
}, 3000)
|
| 32 |
}
|
| 33 |
},
|
| 34 |
(err) => {
|
| 35 |
+
console.error('Voice call error:', err)
|
| 36 |
if (voiceCallActiveRef.current) {
|
| 37 |
+
setText(`\u8bed\u97f3\u542f\u52a8\u5931\u8d25: ${err}`)
|
| 38 |
setTimeout(() => {
|
| 39 |
setText('')
|
| 40 |
+
endVoiceCall()
|
| 41 |
}, 3000)
|
| 42 |
}
|
| 43 |
},
|
| 44 |
language === 'en' ? 'en' : 'zh',
|
| 45 |
)
|
| 46 |
setRecording(true)
|
| 47 |
+
}, [sttProvider, serverStt, language])
|
| 48 |
|
|
|
|
| 49 |
const endVoiceCall = useCallback(async () => {
|
| 50 |
+
if (!voiceCallActiveRef.current) return
|
| 51 |
voiceCallActiveRef.current = false
|
| 52 |
setVoiceCallActive(false)
|
| 53 |
setRecording(false)
|
|
|
|
|
|
|
| 54 |
const finalText = await serverStt.stopStreaming()
|
|
|
|
|
|
|
|
|
|
| 55 |
if (finalText.trim()) {
|
| 56 |
+
// Send text via chat endpoint
|
| 57 |
+
fetch(`${FRIEND_API}/chat`, {
|
| 58 |
+
method: 'POST',
|
| 59 |
+
headers: { 'Content-Type': 'application/json' },
|
| 60 |
+
body: JSON.stringify({ message: finalText }),
|
| 61 |
+
}).catch(() => {})
|
| 62 |
}
|
|
|
|
|
|
|
|
|
|
| 63 |
setText('')
|
| 64 |
+
}, [serverStt])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 65 |
|
| 66 |
// Global keyboard shortcuts
|
| 67 |
useEffect(() => {
|
| 68 |
const onGlobalKeyDown = (e: KeyboardEvent) => {
|
| 69 |
+
if (e.target instanceof HTMLInputElement || e.target instanceof HTMLTextAreaElement) return
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 70 |
|
| 71 |
if (e.key === 'F2') {
|
| 72 |
e.preventDefault()
|
|
|
|
| 79 |
}
|
| 80 |
window.addEventListener('keydown', onGlobalKeyDown)
|
| 81 |
return () => window.removeEventListener('keydown', onGlobalKeyDown)
|
| 82 |
+
}, [voiceCallActive, startVoiceCall, endVoiceCall])
|
| 83 |
|
| 84 |
+
// Broadcast recording state for VRM "listening" response
|
| 85 |
+
useEffect(() => {
|
| 86 |
+
(window as any).__userRecording = recording
|
| 87 |
+
return () => { (window as any).__userRecording = false }
|
| 88 |
+
}, [recording])
|
| 89 |
|
| 90 |
+
// Cleanup display timer
|
| 91 |
+
useEffect(() => {
|
| 92 |
+
return () => {
|
| 93 |
+
if (displayTimerRef.current) clearTimeout(displayTimerRef.current)
|
| 94 |
+
}
|
| 95 |
+
}, [])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 96 |
|
| 97 |
if (voiceCallActive) {
|
| 98 |
return (
|
| 99 |
<div style={barStyle}>
|
| 100 |
<div style={{ flex: 1, position: 'relative' }}>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 101 |
<div
|
| 102 |
style={{
|
| 103 |
...inputStyle,
|
|
|
|
| 115 |
</div>
|
| 116 |
<span
|
| 117 |
style={{
|
| 118 |
+
position: 'absolute',
|
| 119 |
+
top: '50%',
|
| 120 |
+
transform: 'translateY(-50%)',
|
| 121 |
right: 44,
|
| 122 |
+
width: 32,
|
| 123 |
+
height: 32,
|
| 124 |
+
display: 'flex',
|
| 125 |
+
alignItems: 'center',
|
| 126 |
+
justifyContent: 'center',
|
| 127 |
color: recording ? 'rgba(255, 80, 80, 0.9)' : 'rgba(80, 200, 120, 0.9)',
|
| 128 |
animation: recording ? 'claw-pulse 0.8s ease-in-out infinite' : 'none',
|
| 129 |
transition: 'color 0.2s',
|
|
|
|
| 134 |
</span>
|
| 135 |
<button
|
| 136 |
onClick={endVoiceCall}
|
| 137 |
+
style={{
|
| 138 |
+
position: 'absolute',
|
| 139 |
+
top: '50%',
|
| 140 |
+
transform: 'translateY(-50%)',
|
| 141 |
+
right: 12,
|
| 142 |
+
width: 32,
|
| 143 |
+
height: 32,
|
| 144 |
+
border: 'none',
|
| 145 |
+
borderRadius: 4,
|
| 146 |
+
background: 'transparent',
|
| 147 |
+
cursor: 'pointer',
|
| 148 |
+
display: 'flex',
|
| 149 |
+
alignItems: 'center',
|
| 150 |
+
justifyContent: 'center',
|
| 151 |
+
padding: 0,
|
| 152 |
+
color: 'rgba(255, 80, 80, 0.9)',
|
| 153 |
+
}}
|
| 154 |
title={t('挂断 (F2)', 'Hang up (F2)')}
|
| 155 |
>
|
| 156 |
<PhoneOff size={22} />
|
|
|
|
| 161 |
}
|
| 162 |
|
| 163 |
return (
|
| 164 |
+
<div style={{ position: 'absolute', bottom: 8, ...(uiAlign === 'left' ? { left: 12 } : { right: 12 }), zIndex: 300, pointerEvents: 'auto', display: 'flex', flexDirection: 'column', gap: 4, alignItems: 'center' }}>
|
| 165 |
+
<button
|
| 166 |
+
onClick={startVoiceCall}
|
| 167 |
+
style={{
|
| 168 |
+
width: 32,
|
| 169 |
+
height: 32,
|
| 170 |
+
border: 'none',
|
| 171 |
+
borderRadius: 6,
|
| 172 |
+
background: 'rgba(125, 125, 125, 0.28)',
|
| 173 |
+
backdropFilter: 'blur(6px)',
|
| 174 |
+
color: 'rgba(255, 255, 255, 0.8)',
|
| 175 |
+
fontSize: 16,
|
| 176 |
+
cursor: 'pointer',
|
| 177 |
+
display: 'flex',
|
| 178 |
+
alignItems: 'center',
|
| 179 |
+
justifyContent: 'center',
|
| 180 |
+
zIndex: 300,
|
| 181 |
+
pointerEvents: 'auto',
|
| 182 |
+
}}
|
| 183 |
+
title={t('语音通话 (F2)', 'Voice Call (F2)')}
|
| 184 |
+
>
|
| 185 |
+
<Phone size={16} />
|
| 186 |
+
</button>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 187 |
</div>
|
|
|
|
| 188 |
)
|
| 189 |
}
|
| 190 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 191 |
const barStyle: React.CSSProperties = {
|
| 192 |
position: 'absolute',
|
| 193 |
bottom: 16,
|
|
|
|
| 202 |
const inputStyle: React.CSSProperties = {
|
| 203 |
width: '100%',
|
| 204 |
height: 50,
|
| 205 |
+
boxSizing: 'border-box',
|
| 206 |
border: '1px solid rgba(255, 255, 255, 0.2)',
|
| 207 |
borderRadius: 25,
|
| 208 |
background: 'rgba(0, 0, 0, 0.4)',
|
|
|
|
| 213 |
outline: 'none',
|
| 214 |
fontFamily: '"Segoe UI", "Microsoft YaHei", sans-serif',
|
| 215 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
src/components/friend/frontend/components/HistoryPanel.tsx
DELETED
|
@@ -1,177 +0,0 @@
|
|
| 1 |
-
import { useState, useEffect, useRef, useCallback } from 'react'
|
| 2 |
-
import { X } from 'lucide-react'
|
| 3 |
-
import { FRIEND_API } from '../api'
|
| 4 |
-
|
| 5 |
-
interface HistoryMessage {
|
| 6 |
-
role: string
|
| 7 |
-
content: string
|
| 8 |
-
timestamp?: number
|
| 9 |
-
}
|
| 10 |
-
|
| 11 |
-
interface HistoryPanelProps {
|
| 12 |
-
visible: boolean
|
| 13 |
-
onClose: () => void
|
| 14 |
-
language?: 'zh' | 'en'
|
| 15 |
-
}
|
| 16 |
-
|
| 17 |
-
export function HistoryPanel({ visible, onClose, language = 'zh' }: HistoryPanelProps) {
|
| 18 |
-
const t = (zh: string, en: string) => language === 'en' ? en : zh
|
| 19 |
-
const [messages, setMessages] = useState<HistoryMessage[]>([])
|
| 20 |
-
const [agentName, setAgentName] = useState('')
|
| 21 |
-
const [loading, setLoading] = useState(false)
|
| 22 |
-
const scrollRef = useRef<HTMLDivElement>(null)
|
| 23 |
-
|
| 24 |
-
// Drag state
|
| 25 |
-
const [panelPos, setPanelPos] = useState<{ x: number; y: number }>({ x: 0, y: 0 })
|
| 26 |
-
const dragRef = useRef<{ dragging: boolean; startX: number; startY: number; origX: number; origY: number }>({
|
| 27 |
-
dragging: false, startX: 0, startY: 0, origX: 0, origY: 0,
|
| 28 |
-
})
|
| 29 |
-
|
| 30 |
-
const onDragStart = useCallback((e: React.MouseEvent) => {
|
| 31 |
-
dragRef.current = { dragging: true, startX: e.clientX, startY: e.clientY, origX: panelPos.x, origY: panelPos.y }
|
| 32 |
-
const onMove = (ev: MouseEvent) => {
|
| 33 |
-
if (!dragRef.current.dragging) return
|
| 34 |
-
setPanelPos({
|
| 35 |
-
x: dragRef.current.origX + ev.clientX - dragRef.current.startX,
|
| 36 |
-
y: dragRef.current.origY + ev.clientY - dragRef.current.startY,
|
| 37 |
-
})
|
| 38 |
-
}
|
| 39 |
-
const onUp = () => {
|
| 40 |
-
dragRef.current.dragging = false
|
| 41 |
-
window.removeEventListener('mousemove', onMove)
|
| 42 |
-
window.removeEventListener('mouseup', onUp)
|
| 43 |
-
}
|
| 44 |
-
window.addEventListener('mousemove', onMove)
|
| 45 |
-
window.addEventListener('mouseup', onUp)
|
| 46 |
-
}, [panelPos])
|
| 47 |
-
|
| 48 |
-
useEffect(() => {
|
| 49 |
-
if (visible) setPanelPos({ x: 0, y: 0 })
|
| 50 |
-
}, [visible])
|
| 51 |
-
|
| 52 |
-
// Fetch history when panel opens
|
| 53 |
-
useEffect(() => {
|
| 54 |
-
if (!visible) return
|
| 55 |
-
setLoading(true)
|
| 56 |
-
fetch(`${FRIEND_API}/history`)
|
| 57 |
-
.then((r) => r.json())
|
| 58 |
-
.then((data) => {
|
| 59 |
-
setMessages(data.messages || [])
|
| 60 |
-
if (data.agentName) setAgentName(data.agentName)
|
| 61 |
-
setTimeout(() => {
|
| 62 |
-
if (scrollRef.current) scrollRef.current.scrollTop = scrollRef.current.scrollHeight
|
| 63 |
-
}, 50)
|
| 64 |
-
})
|
| 65 |
-
.catch(() => setMessages([]))
|
| 66 |
-
.finally(() => setLoading(false))
|
| 67 |
-
}, [visible])
|
| 68 |
-
|
| 69 |
-
if (!visible) return null
|
| 70 |
-
|
| 71 |
-
return (
|
| 72 |
-
<div style={overlayStyle} data-no-passthrough onClick={onClose}>
|
| 73 |
-
<div style={{ ...panelStyle, transform: `translate(${panelPos.x}px, ${panelPos.y}px)` }} data-no-passthrough onClick={(e) => e.stopPropagation()}>
|
| 74 |
-
<div style={headerStyle} onMouseDown={onDragStart}>
|
| 75 |
-
<span style={{ fontSize: 16, fontWeight: 600, cursor: 'grab' }}>{t('对话历史', 'Chat History')}</span>
|
| 76 |
-
<button onClick={onClose} style={closeBtnStyle}>
|
| 77 |
-
<X size={16} />
|
| 78 |
-
</button>
|
| 79 |
-
</div>
|
| 80 |
-
|
| 81 |
-
<div ref={scrollRef} style={messagesContainerStyle}>
|
| 82 |
-
{loading && (
|
| 83 |
-
<div style={{ textAlign: 'center', color: 'rgba(255,255,255,0.4)', padding: 20 }}>{t('加载中...', 'Loading...')}</div>
|
| 84 |
-
)}
|
| 85 |
-
{!loading && messages.length === 0 && (
|
| 86 |
-
<div style={{ textAlign: 'center', color: 'rgba(255,255,255,0.4)', padding: 20 }}>{t('暂无对话记录', 'No chat history')}</div>
|
| 87 |
-
)}
|
| 88 |
-
{messages.map((msg, i) => (
|
| 89 |
-
<div key={i} style={{ display: 'flex', flexDirection: 'column', alignItems: msg.role === 'user' ? 'flex-end' : 'flex-start' }}>
|
| 90 |
-
<div style={{ fontSize: 11, color: 'rgba(255,255,255,0.35)', marginBottom: 2 }}>
|
| 91 |
-
{msg.role === 'user' ? t('你', 'You') : (agentName || 'AI')}
|
| 92 |
-
</div>
|
| 93 |
-
<div style={{
|
| 94 |
-
...bubbleStyle,
|
| 95 |
-
background: msg.role === 'user' ? 'rgba(100, 160, 255, 0.3)' : 'rgba(255, 255, 255, 0.08)',
|
| 96 |
-
borderColor: msg.role === 'user' ? 'rgba(100, 160, 255, 0.4)' : 'rgba(255, 255, 255, 0.12)',
|
| 97 |
-
}}>
|
| 98 |
-
{msg.content}
|
| 99 |
-
</div>
|
| 100 |
-
</div>
|
| 101 |
-
))}
|
| 102 |
-
</div>
|
| 103 |
-
</div>
|
| 104 |
-
</div>
|
| 105 |
-
)
|
| 106 |
-
}
|
| 107 |
-
|
| 108 |
-
const overlayStyle: React.CSSProperties = {
|
| 109 |
-
position: 'fixed',
|
| 110 |
-
top: 0,
|
| 111 |
-
left: 0,
|
| 112 |
-
right: 0,
|
| 113 |
-
bottom: 0,
|
| 114 |
-
background: 'rgba(0, 0, 0, 0.5)',
|
| 115 |
-
zIndex: 1000,
|
| 116 |
-
display: 'flex',
|
| 117 |
-
alignItems: 'center',
|
| 118 |
-
justifyContent: 'center',
|
| 119 |
-
pointerEvents: 'auto',
|
| 120 |
-
}
|
| 121 |
-
|
| 122 |
-
const panelStyle: React.CSSProperties = {
|
| 123 |
-
width: 360,
|
| 124 |
-
maxHeight: '80vh',
|
| 125 |
-
background: 'rgba(30, 30, 40, 0.95)',
|
| 126 |
-
backdropFilter: 'blur(12px)',
|
| 127 |
-
borderRadius: 12,
|
| 128 |
-
border: '1px solid rgba(255, 255, 255, 0.15)',
|
| 129 |
-
boxShadow: '0 8px 32px rgba(0, 0, 0, 0.5)',
|
| 130 |
-
padding: 16,
|
| 131 |
-
color: '#fff',
|
| 132 |
-
fontFamily: '"Segoe UI", "Microsoft YaHei", sans-serif',
|
| 133 |
-
display: 'flex',
|
| 134 |
-
flexDirection: 'column',
|
| 135 |
-
}
|
| 136 |
-
|
| 137 |
-
const headerStyle: React.CSSProperties = {
|
| 138 |
-
display: 'flex',
|
| 139 |
-
justifyContent: 'space-between',
|
| 140 |
-
alignItems: 'center',
|
| 141 |
-
marginBottom: 12,
|
| 142 |
-
cursor: 'grab',
|
| 143 |
-
userSelect: 'none',
|
| 144 |
-
}
|
| 145 |
-
|
| 146 |
-
const closeBtnStyle: React.CSSProperties = {
|
| 147 |
-
width: 28,
|
| 148 |
-
height: 28,
|
| 149 |
-
border: 'none',
|
| 150 |
-
borderRadius: 6,
|
| 151 |
-
background: 'rgba(255, 255, 255, 0.1)',
|
| 152 |
-
color: 'rgba(255, 255, 255, 0.7)',
|
| 153 |
-
cursor: 'pointer',
|
| 154 |
-
display: 'flex',
|
| 155 |
-
alignItems: 'center',
|
| 156 |
-
justifyContent: 'center',
|
| 157 |
-
}
|
| 158 |
-
|
| 159 |
-
const messagesContainerStyle: React.CSSProperties = {
|
| 160 |
-
flex: 1,
|
| 161 |
-
overflowY: 'auto',
|
| 162 |
-
display: 'flex',
|
| 163 |
-
flexDirection: 'column',
|
| 164 |
-
gap: 12,
|
| 165 |
-
maxHeight: 'calc(80vh - 60px)',
|
| 166 |
-
}
|
| 167 |
-
|
| 168 |
-
const bubbleStyle: React.CSSProperties = {
|
| 169 |
-
maxWidth: '85%',
|
| 170 |
-
padding: '8px 12px',
|
| 171 |
-
borderRadius: 10,
|
| 172 |
-
border: '1px solid',
|
| 173 |
-
fontSize: 13,
|
| 174 |
-
lineHeight: 1.5,
|
| 175 |
-
wordBreak: 'break-word',
|
| 176 |
-
whiteSpace: 'pre-wrap',
|
| 177 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
src/components/friend/frontend/components/MoodIndicator.tsx
DELETED
|
@@ -1,372 +0,0 @@
|
|
| 1 |
-
import { FRIEND_API } from '../api'
|
| 2 |
-
import { useState, useEffect, useRef, useCallback } from 'react'
|
| 3 |
-
|
| 4 |
-
interface MoodBubble {
|
| 5 |
-
id: number
|
| 6 |
-
delta: number
|
| 7 |
-
}
|
| 8 |
-
|
| 9 |
-
interface MoodIndicatorProps {
|
| 10 |
-
uiAlign?: 'left' | 'right'
|
| 11 |
-
}
|
| 12 |
-
|
| 13 |
-
const BAR_WIDTH = 10
|
| 14 |
-
const BAR_HEIGHT = 120
|
| 15 |
-
const BORDER_GAP = 4 // gap between bar and border
|
| 16 |
-
const BORDER_WIDTH = 2 // border stroke thickness
|
| 17 |
-
const HEART_SIZE = 23
|
| 18 |
-
const DPR = typeof window !== 'undefined' ? (window.devicePixelRatio || 1) : 1
|
| 19 |
-
|
| 20 |
-
// Wave parameters (ported from Android HorizontalWaveProgressView)
|
| 21 |
-
const WAVE_LENGTH = 8
|
| 22 |
-
const WAVE_HEIGHT = 2.5
|
| 23 |
-
|
| 24 |
-
// 5 tiers: 90+ pink, 70+ orange, 50+ green, 30+ blue, 0+ grey
|
| 25 |
-
function moodRGB(percent: number): [number, number, number] {
|
| 26 |
-
if (percent >= 90) return [255, 107, 157]
|
| 27 |
-
if (percent >= 70) return [255, 165, 70]
|
| 28 |
-
if (percent >= 50) return [72, 199, 142]
|
| 29 |
-
if (percent >= 30) return [78, 168, 222]
|
| 30 |
-
return [160, 168, 180]
|
| 31 |
-
}
|
| 32 |
-
|
| 33 |
-
/** Rounded-rect path helper */
|
| 34 |
-
function roundedRectPath(ctx: CanvasRenderingContext2D, x: number, y: number, w: number, h: number, r: number) {
|
| 35 |
-
ctx.beginPath()
|
| 36 |
-
ctx.moveTo(x + r, y)
|
| 37 |
-
ctx.lineTo(x + w - r, y)
|
| 38 |
-
ctx.arc(x + w - r, y + r, r, -Math.PI / 2, 0)
|
| 39 |
-
ctx.lineTo(x + w, y + h - r)
|
| 40 |
-
ctx.arc(x + w - r, y + h - r, r, 0, Math.PI / 2)
|
| 41 |
-
ctx.lineTo(x + r, y + h)
|
| 42 |
-
ctx.arc(x + r, y + h - r, r, Math.PI / 2, Math.PI)
|
| 43 |
-
ctx.lineTo(x, y + r)
|
| 44 |
-
ctx.arc(x + r, y + r, r, Math.PI, Math.PI * 1.5)
|
| 45 |
-
ctx.closePath()
|
| 46 |
-
}
|
| 47 |
-
|
| 48 |
-
/** Heart path centered at (cx, cy) with given size — classic round heart */
|
| 49 |
-
function heartPath(ctx: CanvasRenderingContext2D, cx: number, cy: number, size: number) {
|
| 50 |
-
const r = size * 0.45
|
| 51 |
-
const topY = cy - r * 0.4
|
| 52 |
-
ctx.beginPath()
|
| 53 |
-
// Bottom tip (rounded)
|
| 54 |
-
ctx.moveTo(cx, cy + r * 0.85)
|
| 55 |
-
// Left curve
|
| 56 |
-
ctx.bezierCurveTo(cx - r * 0.5, cy + r * 0.6, cx - r * 1.1, cy + r * 0.1, cx - r, topY)
|
| 57 |
-
// Left bump (arc)
|
| 58 |
-
ctx.arc(cx - r * 0.5, topY, r * 0.5, Math.PI, 0, false)
|
| 59 |
-
// Right bump (arc)
|
| 60 |
-
ctx.arc(cx + r * 0.5, topY, r * 0.5, Math.PI, 0, false)
|
| 61 |
-
// Right curve
|
| 62 |
-
ctx.bezierCurveTo(cx + r * 1.1, cy + r * 0.1, cx + r * 0.5, cy + r * 0.6, cx, cy + r * 0.85)
|
| 63 |
-
ctx.closePath()
|
| 64 |
-
}
|
| 65 |
-
|
| 66 |
-
/**
|
| 67 |
-
* Common wave drawing logic used by both the bar and the heart.
|
| 68 |
-
* Draws dual bezier waves inside the current clip region.
|
| 69 |
-
*/
|
| 70 |
-
function drawWaves(
|
| 71 |
-
ctx: CanvasRenderingContext2D,
|
| 72 |
-
regionWidth: number,
|
| 73 |
-
regionHeight: number,
|
| 74 |
-
waterY: number,
|
| 75 |
-
moveDistance: number,
|
| 76 |
-
cr: number, cg: number, cb: number,
|
| 77 |
-
) {
|
| 78 |
-
const fullCycle = WAVE_LENGTH * 2
|
| 79 |
-
const waveNumber = Math.ceil(regionWidth / fullCycle) + 2
|
| 80 |
-
|
| 81 |
-
const drawWave = (scrollDir: number, alpha: number) => {
|
| 82 |
-
ctx.beginPath()
|
| 83 |
-
const offset = (moveDistance * scrollDir) % fullCycle
|
| 84 |
-
let x = -fullCycle + offset
|
| 85 |
-
ctx.moveTo(x, waterY)
|
| 86 |
-
|
| 87 |
-
for (let i = 0; i < waveNumber * 2; i++) {
|
| 88 |
-
ctx.quadraticCurveTo(x + WAVE_LENGTH / 2, waterY - WAVE_HEIGHT, x + WAVE_LENGTH, waterY)
|
| 89 |
-
x += WAVE_LENGTH
|
| 90 |
-
ctx.quadraticCurveTo(x + WAVE_LENGTH / 2, waterY + WAVE_HEIGHT, x + WAVE_LENGTH, waterY)
|
| 91 |
-
x += WAVE_LENGTH
|
| 92 |
-
}
|
| 93 |
-
|
| 94 |
-
ctx.lineTo(x, regionHeight)
|
| 95 |
-
ctx.lineTo(-fullCycle, regionHeight)
|
| 96 |
-
ctx.closePath()
|
| 97 |
-
ctx.fillStyle = `rgba(${cr},${cg},${cb},${alpha})`
|
| 98 |
-
ctx.fill()
|
| 99 |
-
}
|
| 100 |
-
|
| 101 |
-
drawWave(-1, 0.55)
|
| 102 |
-
drawWave(1, 1.0)
|
| 103 |
-
}
|
| 104 |
-
|
| 105 |
-
/**
|
| 106 |
-
* Draw the vertical liquid-filled bar with border that has a gap.
|
| 107 |
-
*/
|
| 108 |
-
function drawLiquidBar(
|
| 109 |
-
ctx: CanvasRenderingContext2D,
|
| 110 |
-
w: number,
|
| 111 |
-
h: number,
|
| 112 |
-
percent: number,
|
| 113 |
-
moveDistance: number,
|
| 114 |
-
) {
|
| 115 |
-
const radius = w / 2
|
| 116 |
-
const totalW = w + (BORDER_GAP + BORDER_WIDTH) * 2
|
| 117 |
-
const totalH = h + (BORDER_GAP + BORDER_WIDTH) * 2
|
| 118 |
-
const offsetX = BORDER_GAP + BORDER_WIDTH
|
| 119 |
-
const offsetY = BORDER_GAP + BORDER_WIDTH
|
| 120 |
-
|
| 121 |
-
ctx.clearRect(0, 0, totalW, totalH)
|
| 122 |
-
|
| 123 |
-
const waterY = h * (1 - percent / 100)
|
| 124 |
-
const [cr, cg, cb] = moodRGB(percent)
|
| 125 |
-
|
| 126 |
-
// ── Bar fill ──
|
| 127 |
-
ctx.save()
|
| 128 |
-
roundedRectPath(ctx, offsetX, offsetY, w, h, radius)
|
| 129 |
-
ctx.clip()
|
| 130 |
-
|
| 131 |
-
ctx.clearRect(offsetX, offsetY, w, h)
|
| 132 |
-
|
| 133 |
-
ctx.save()
|
| 134 |
-
ctx.translate(offsetX, offsetY)
|
| 135 |
-
drawWaves(ctx, w, h, waterY, moveDistance, cr, cg, cb)
|
| 136 |
-
ctx.restore()
|
| 137 |
-
|
| 138 |
-
ctx.restore()
|
| 139 |
-
|
| 140 |
-
// ── Border with gap ──
|
| 141 |
-
const bx = offsetX - BORDER_GAP - BORDER_WIDTH / 2
|
| 142 |
-
const by = offsetY - BORDER_GAP - BORDER_WIDTH / 2
|
| 143 |
-
const bw = w + (BORDER_GAP + BORDER_WIDTH / 2) * 2
|
| 144 |
-
const bh = h + (BORDER_GAP + BORDER_WIDTH / 2) * 2
|
| 145 |
-
const br = radius + BORDER_GAP + BORDER_WIDTH / 2
|
| 146 |
-
roundedRectPath(ctx, bx, by, bw, bh, br)
|
| 147 |
-
ctx.strokeStyle = 'rgba(80, 80, 85, 0.75)'
|
| 148 |
-
ctx.lineWidth = BORDER_WIDTH
|
| 149 |
-
ctx.stroke()
|
| 150 |
-
}
|
| 151 |
-
|
| 152 |
-
/**
|
| 153 |
-
* Draw liquid-filled heart icon with border and gap (matching bar style).
|
| 154 |
-
*/
|
| 155 |
-
function drawLiquidHeart(
|
| 156 |
-
ctx: CanvasRenderingContext2D,
|
| 157 |
-
size: number,
|
| 158 |
-
percent: number,
|
| 159 |
-
moveDistance: number,
|
| 160 |
-
) {
|
| 161 |
-
ctx.clearRect(0, 0, size, size)
|
| 162 |
-
|
| 163 |
-
const cx = size / 2
|
| 164 |
-
const cy = size / 2
|
| 165 |
-
const waterY = size * (1 - percent / 100)
|
| 166 |
-
const [cr, cg, cb] = moodRGB(percent)
|
| 167 |
-
|
| 168 |
-
// ── Heart fill ──
|
| 169 |
-
ctx.save()
|
| 170 |
-
heartPath(ctx, cx, cy, size)
|
| 171 |
-
ctx.clip()
|
| 172 |
-
|
| 173 |
-
ctx.fillStyle = 'rgba(60, 60, 65, 0.9)'
|
| 174 |
-
ctx.fillRect(0, 0, size, size)
|
| 175 |
-
|
| 176 |
-
drawWaves(ctx, size, size, waterY, moveDistance, cr, cg, cb)
|
| 177 |
-
|
| 178 |
-
ctx.restore()
|
| 179 |
-
}
|
| 180 |
-
|
| 181 |
-
let bubbleIdCounter = 0
|
| 182 |
-
|
| 183 |
-
// Canvas total dimensions (bar + border gap)
|
| 184 |
-
const BAR_CANVAS_W = BAR_WIDTH + (BORDER_GAP + BORDER_WIDTH) * 2
|
| 185 |
-
const BAR_CANVAS_H = BAR_HEIGHT + (BORDER_GAP + BORDER_WIDTH) * 2
|
| 186 |
-
|
| 187 |
-
export function MoodIndicator({ uiAlign = 'right' }: MoodIndicatorProps) {
|
| 188 |
-
const [mood, setMood] = useState(60)
|
| 189 |
-
const [bubbles, setBubbles] = useState<MoodBubble[]>([])
|
| 190 |
-
const [visible, setVisible] = useState(false)
|
| 191 |
-
const [dragging, setDragging] = useState(false)
|
| 192 |
-
const defaultX = (align: string) => align === 'left' ? window.innerWidth - 20 - BAR_CANVAS_W : 20
|
| 193 |
-
const [pos, setPos] = useState<{ x: number; y: number }>({ x: defaultX(uiAlign), y: 20 })
|
| 194 |
-
const [userDragged, setUserDragged] = useState(false)
|
| 195 |
-
const hideTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null)
|
| 196 |
-
const containerRef = useRef<HTMLDivElement>(null)
|
| 197 |
-
const barCanvasRef = useRef<HTMLCanvasElement>(null)
|
| 198 |
-
const heartCanvasRef = useRef<HTMLCanvasElement>(null)
|
| 199 |
-
const animRef = useRef<number>(0)
|
| 200 |
-
const displayPercentRef = useRef(60)
|
| 201 |
-
const moveDistRef = useRef(0)
|
| 202 |
-
|
| 203 |
-
useEffect(() => {
|
| 204 |
-
if (!userDragged) setPos(p => ({ ...p, x: defaultX(uiAlign) }))
|
| 205 |
-
}, [uiAlign, userDragged])
|
| 206 |
-
|
| 207 |
-
useEffect(() => {
|
| 208 |
-
fetch(`${FRIEND_API}/settings`)
|
| 209 |
-
.then((r) => r.json())
|
| 210 |
-
.then((s) => { if (s.moodIndex !== undefined) { setMood(s.moodIndex); displayPercentRef.current = s.moodIndex } })
|
| 211 |
-
.catch(() => {})
|
| 212 |
-
}, [])
|
| 213 |
-
|
| 214 |
-
const showBriefly = useCallback(() => {
|
| 215 |
-
setVisible(true)
|
| 216 |
-
if (hideTimerRef.current) clearTimeout(hideTimerRef.current)
|
| 217 |
-
hideTimerRef.current = setTimeout(() => setVisible(false), 5000)
|
| 218 |
-
}, [])
|
| 219 |
-
|
| 220 |
-
useEffect(() => {
|
| 221 |
-
const es = new EventSource(`${FRIEND_API}/events`)
|
| 222 |
-
es.onmessage = (e) => {
|
| 223 |
-
try {
|
| 224 |
-
const data = JSON.parse(e.data)
|
| 225 |
-
if (data.moodIndex !== undefined && data.moodDelta !== undefined) {
|
| 226 |
-
setMood(data.moodIndex)
|
| 227 |
-
showBriefly()
|
| 228 |
-
const id = ++bubbleIdCounter
|
| 229 |
-
setBubbles((prev) => [...prev, { id, delta: data.moodDelta }])
|
| 230 |
-
setTimeout(() => setBubbles((prev) => prev.filter((b) => b.id !== id)), 2000)
|
| 231 |
-
}
|
| 232 |
-
} catch { /* ignore */ }
|
| 233 |
-
}
|
| 234 |
-
return () => es.close()
|
| 235 |
-
}, [showBriefly])
|
| 236 |
-
|
| 237 |
-
// Canvas animation loop — draws both heart and bar
|
| 238 |
-
useEffect(() => {
|
| 239 |
-
const barCanvas = barCanvasRef.current
|
| 240 |
-
const heartCanvas = heartCanvasRef.current
|
| 241 |
-
if (!barCanvas || !heartCanvas) return
|
| 242 |
-
const barCtx = barCanvas.getContext('2d')
|
| 243 |
-
const heartCtx = heartCanvas.getContext('2d')
|
| 244 |
-
if (!barCtx || !heartCtx) return
|
| 245 |
-
|
| 246 |
-
barCanvas.width = BAR_CANVAS_W * DPR
|
| 247 |
-
barCanvas.height = BAR_CANVAS_H * DPR
|
| 248 |
-
barCtx.scale(DPR, DPR)
|
| 249 |
-
|
| 250 |
-
heartCanvas.width = HEART_SIZE * DPR
|
| 251 |
-
heartCanvas.height = HEART_SIZE * DPR
|
| 252 |
-
heartCtx.scale(DPR, DPR)
|
| 253 |
-
|
| 254 |
-
let running = true
|
| 255 |
-
const animate = () => {
|
| 256 |
-
if (!running) return
|
| 257 |
-
const target = Math.max(2, mood)
|
| 258 |
-
displayPercentRef.current += (target - displayPercentRef.current) * 0.08
|
| 259 |
-
moveDistRef.current += 0.12
|
| 260 |
-
|
| 261 |
-
drawLiquidBar(barCtx, BAR_WIDTH, BAR_HEIGHT, displayPercentRef.current, moveDistRef.current)
|
| 262 |
-
drawLiquidHeart(heartCtx, HEART_SIZE, displayPercentRef.current, moveDistRef.current)
|
| 263 |
-
animRef.current = requestAnimationFrame(animate)
|
| 264 |
-
}
|
| 265 |
-
animate()
|
| 266 |
-
return () => { running = false; cancelAnimationFrame(animRef.current) }
|
| 267 |
-
}, [mood])
|
| 268 |
-
|
| 269 |
-
const handlePointerDown = useCallback((e: React.PointerEvent) => {
|
| 270 |
-
e.preventDefault()
|
| 271 |
-
e.stopPropagation()
|
| 272 |
-
setDragging(true)
|
| 273 |
-
setUserDragged(true)
|
| 274 |
-
setVisible(true)
|
| 275 |
-
if (hideTimerRef.current) { clearTimeout(hideTimerRef.current); hideTimerRef.current = null }
|
| 276 |
-
|
| 277 |
-
const el = containerRef.current
|
| 278 |
-
if (!el) return
|
| 279 |
-
const rect = el.getBoundingClientRect()
|
| 280 |
-
const offsetX = e.clientX - rect.left
|
| 281 |
-
const offsetY = e.clientY - rect.top
|
| 282 |
-
|
| 283 |
-
const onMove = (ev: PointerEvent) => {
|
| 284 |
-
setPos({ x: ev.clientX - offsetX, y: ev.clientY - offsetY })
|
| 285 |
-
}
|
| 286 |
-
const onUp = () => {
|
| 287 |
-
setDragging(false)
|
| 288 |
-
window.removeEventListener('pointermove', onMove)
|
| 289 |
-
window.removeEventListener('pointerup', onUp)
|
| 290 |
-
hideTimerRef.current = setTimeout(() => setVisible(false), 3000)
|
| 291 |
-
}
|
| 292 |
-
window.addEventListener('pointermove', onMove)
|
| 293 |
-
window.addEventListener('pointerup', onUp)
|
| 294 |
-
}, [])
|
| 295 |
-
|
| 296 |
-
const handleMouseEnter = useCallback(() => {
|
| 297 |
-
setVisible(true)
|
| 298 |
-
if (hideTimerRef.current) { clearTimeout(hideTimerRef.current); hideTimerRef.current = null }
|
| 299 |
-
}, [])
|
| 300 |
-
|
| 301 |
-
const handleMouseLeave = useCallback(() => {
|
| 302 |
-
if (!dragging) {
|
| 303 |
-
hideTimerRef.current = setTimeout(() => setVisible(false), 2000)
|
| 304 |
-
}
|
| 305 |
-
}, [dragging])
|
| 306 |
-
|
| 307 |
-
return (
|
| 308 |
-
<div
|
| 309 |
-
ref={containerRef}
|
| 310 |
-
style={{
|
| 311 |
-
position: 'absolute',
|
| 312 |
-
top: pos.y,
|
| 313 |
-
left: pos.x,
|
| 314 |
-
zIndex: 250,
|
| 315 |
-
pointerEvents: 'auto',
|
| 316 |
-
display: 'flex',
|
| 317 |
-
flexDirection: 'column',
|
| 318 |
-
alignItems: 'center',
|
| 319 |
-
transition: 'opacity 0.5s ease',
|
| 320 |
-
opacity: visible || bubbles.length > 0 ? 1 : 0.5,
|
| 321 |
-
cursor: dragging ? 'grabbing' : 'grab',
|
| 322 |
-
userSelect: 'none',
|
| 323 |
-
}}
|
| 324 |
-
onMouseEnter={handleMouseEnter}
|
| 325 |
-
onMouseLeave={handleMouseLeave}
|
| 326 |
-
onPointerDown={handlePointerDown}
|
| 327 |
-
data-no-passthrough
|
| 328 |
-
>
|
| 329 |
-
{/* Liquid-filled heart */}
|
| 330 |
-
<canvas
|
| 331 |
-
ref={heartCanvasRef}
|
| 332 |
-
width={HEART_SIZE * DPR}
|
| 333 |
-
height={HEART_SIZE * DPR}
|
| 334 |
-
style={{ width: HEART_SIZE, height: HEART_SIZE, display: 'block', marginBottom: 4 }}
|
| 335 |
-
/>
|
| 336 |
-
|
| 337 |
-
{/* Vertical liquid bar with border */}
|
| 338 |
-
<div style={{ position: 'relative', width: BAR_CANVAS_W, height: BAR_CANVAS_H }}>
|
| 339 |
-
<canvas
|
| 340 |
-
ref={barCanvasRef}
|
| 341 |
-
width={BAR_CANVAS_W * DPR}
|
| 342 |
-
height={BAR_CANVAS_H * DPR}
|
| 343 |
-
style={{ width: BAR_CANVAS_W, height: BAR_CANVAS_H, display: 'block' }}
|
| 344 |
-
/>
|
| 345 |
-
{/* Floating bubbles */}
|
| 346 |
-
{bubbles.map((b) => (
|
| 347 |
-
<span
|
| 348 |
-
key={b.id}
|
| 349 |
-
style={{
|
| 350 |
-
position: 'absolute',
|
| 351 |
-
left: '50%',
|
| 352 |
-
top: -20,
|
| 353 |
-
transform: 'translateX(-50%)',
|
| 354 |
-
fontSize: 14,
|
| 355 |
-
fontWeight: 800,
|
| 356 |
-
color: b.delta > 0 ? '#FF6B9D' : '#7EB0D5',
|
| 357 |
-
whiteSpace: 'nowrap',
|
| 358 |
-
pointerEvents: 'none',
|
| 359 |
-
animation: 'mood-bubble-float 2.5s ease-out forwards',
|
| 360 |
-
fontFamily: '"Segoe UI", "Microsoft YaHei", sans-serif',
|
| 361 |
-
textShadow: b.delta > 0
|
| 362 |
-
? '0 0 6px rgba(255,107,157,0.6), 0 1px 3px rgba(0,0,0,0.4)'
|
| 363 |
-
: '0 0 6px rgba(126,176,213,0.6), 0 1px 3px rgba(0,0,0,0.4)',
|
| 364 |
-
}}
|
| 365 |
-
>
|
| 366 |
-
{b.delta > 0 ? `❤️+${b.delta}` : `🩶${b.delta}`}
|
| 367 |
-
</span>
|
| 368 |
-
))}
|
| 369 |
-
</div>
|
| 370 |
-
</div>
|
| 371 |
-
)
|
| 372 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
src/components/friend/frontend/components/SettingsPanel.tsx
CHANGED
|
@@ -1,16 +1,6 @@
|
|
| 1 |
import { useState, useEffect, useRef, useCallback } from 'react'
|
| 2 |
-
import { X, Play, Loader
|
| 3 |
import { FRIEND_API } from '../api'
|
| 4 |
-
import { dancePresets, type DancePreset } from '../motion-controller'
|
| 5 |
-
import { pickFile } from '../web-file-picker'
|
| 6 |
-
|
| 7 |
-
interface DanceItem {
|
| 8 |
-
id: string
|
| 9 |
-
label: string
|
| 10 |
-
vmdUrl: string
|
| 11 |
-
bgmUrl?: string
|
| 12 |
-
builtin?: boolean
|
| 13 |
-
}
|
| 14 |
|
| 15 |
type SttProvider = 'browser' | 'groq' | 'anthropic' | 'local' | 'doubao'
|
| 16 |
|
|
@@ -31,23 +21,13 @@ interface SettingsPanelProps {
|
|
| 31 |
onVolumeChange: (v: number) => void
|
| 32 |
uiAlign: 'left' | 'right'
|
| 33 |
onUiAlignChange: (v: 'left' | 'right') => void
|
| 34 |
-
hideMood: boolean
|
| 35 |
-
onHideMoodChange: (v: boolean) => void
|
| 36 |
-
screenObserve: boolean
|
| 37 |
-
onScreenObserveChange: (v: boolean) => void
|
| 38 |
-
screenObserveInterval: number
|
| 39 |
-
onScreenObserveIntervalChange: (v: number) => void
|
| 40 |
-
/** Return a data URL screenshot of the current VRM canvas */
|
| 41 |
-
captureVrmScreenshot?: () => string | null
|
| 42 |
language: 'zh' | 'en'
|
| 43 |
onLanguageChange: (v: 'zh' | 'en') => void
|
| 44 |
-
currentDance: string
|
| 45 |
-
onDanceChange: (id: string, preset?: DancePreset) => void
|
| 46 |
sttProvider?: SttProvider
|
| 47 |
onSttProviderChange?: (v: SttProvider) => void
|
| 48 |
}
|
| 49 |
|
| 50 |
-
type Tab = 'general' | 'voice' | 'model'
|
| 51 |
|
| 52 |
const BUILTIN_MODELS = ['/friend/model1.vrm', '/friend/model2.vrm', '/friend/model3.vrm', '/friend/model4.vrm', '/friend/model5.vrm']
|
| 53 |
|
|
@@ -103,32 +83,18 @@ export function SettingsPanel({
|
|
| 103 |
tracking, onTrackingChange,
|
| 104 |
volume, onVolumeChange,
|
| 105 |
uiAlign, onUiAlignChange,
|
| 106 |
-
hideMood, onHideMoodChange,
|
| 107 |
-
screenObserve, onScreenObserveChange,
|
| 108 |
-
screenObserveInterval, onScreenObserveIntervalChange,
|
| 109 |
-
captureVrmScreenshot,
|
| 110 |
language, onLanguageChange,
|
| 111 |
-
currentDance, onDanceChange,
|
| 112 |
sttProvider = 'browser', onSttProviderChange,
|
| 113 |
}: SettingsPanelProps) {
|
| 114 |
const t = (zh: string, en: string) => language === 'en' ? en : zh
|
| 115 |
|
| 116 |
const [tab, setTab] = useState<Tab>('general')
|
| 117 |
-
const [models, setModels] = useState<string[]>([])
|
| 118 |
-
const [soulContent, setSoulContent] = useState('')
|
| 119 |
-
const [identityContent, setIdentityContent] = useState('')
|
| 120 |
-
const [personaDirty, setPersonaDirty] = useState(false)
|
| 121 |
-
const [personaSaving, setPersonaSaving] = useState(false)
|
| 122 |
-
const [generating, setGenerating] = useState(false)
|
| 123 |
const [currentVoice, setCurrentVoice] = useState('')
|
| 124 |
const [currentProvider, setCurrentProvider] = useState<string>('edge')
|
| 125 |
const [qwenKey, setQwenKey] = useState('')
|
| 126 |
const [qwenModel, setQwenModel] = useState('qwen3-tts-flash')
|
| 127 |
const [previewingId, setPreviewingId] = useState<string | null>(null)
|
| 128 |
const audioRef = useRef<HTMLAudioElement | null>(null)
|
| 129 |
-
const [customDances, setCustomDances] = useState<DanceItem[]>([])
|
| 130 |
-
const [importingDance, setImportingDance] = useState(false)
|
| 131 |
-
|
| 132 |
|
| 133 |
// Drag state
|
| 134 |
const [panelPos, setPanelPos] = useState<{ x: number; y: number }>({ x: 0, y: 0 })
|
|
@@ -154,18 +120,12 @@ export function SettingsPanel({
|
|
| 154 |
window.addEventListener('mouseup', onUp)
|
| 155 |
}, [panelPos])
|
| 156 |
|
| 157 |
-
// Reset position when panel opens
|
| 158 |
useEffect(() => {
|
| 159 |
if (visible) setPanelPos({ x: 0, y: 0 })
|
| 160 |
}, [visible])
|
| 161 |
|
| 162 |
useEffect(() => {
|
| 163 |
if (!visible) return
|
| 164 |
-
fetch(`${FRIEND_API}/model/list`)
|
| 165 |
-
.then((r) => r.json())
|
| 166 |
-
.then((data) => { if (data.models) setModels(data.models) })
|
| 167 |
-
.catch(() => setModels([]))
|
| 168 |
-
// Fetch current voice + provider
|
| 169 |
fetch(`${FRIEND_API}/voice`)
|
| 170 |
.then((r) => r.json())
|
| 171 |
.then((data) => {
|
|
@@ -177,72 +137,6 @@ export function SettingsPanel({
|
|
| 177 |
.catch(() => {})
|
| 178 |
}, [visible])
|
| 179 |
|
| 180 |
-
// Fetch persona files when model tab / persona sub-tab is active
|
| 181 |
-
useEffect(() => {
|
| 182 |
-
if (!visible || tab !== 'persona') return
|
| 183 |
-
fetch(`${FRIEND_API}/persona`)
|
| 184 |
-
.then((r) => r.json())
|
| 185 |
-
.then((data) => {
|
| 186 |
-
setSoulContent(data.soul || '')
|
| 187 |
-
setIdentityContent(data.identity || '')
|
| 188 |
-
setPersonaDirty(false)
|
| 189 |
-
})
|
| 190 |
-
.catch(() => {})
|
| 191 |
-
}, [visible, tab])
|
| 192 |
-
|
| 193 |
-
const fetchCustomDances = useCallback(() => {
|
| 194 |
-
fetch(`${FRIEND_API}/dance/list`)
|
| 195 |
-
.then((r) => r.json())
|
| 196 |
-
.then((data) => { if (data.dances) setCustomDances(data.dances) })
|
| 197 |
-
.catch(() => setCustomDances([]))
|
| 198 |
-
}, [])
|
| 199 |
-
|
| 200 |
-
useEffect(() => {
|
| 201 |
-
if (!visible || tab !== 'dance') return
|
| 202 |
-
fetchCustomDances()
|
| 203 |
-
}, [visible, tab, fetchCustomDances])
|
| 204 |
-
|
| 205 |
-
const savePersona = useCallback(() => {
|
| 206 |
-
setPersonaSaving(true)
|
| 207 |
-
fetch(`${FRIEND_API}/persona`, {
|
| 208 |
-
method: 'POST',
|
| 209 |
-
headers: { 'Content-Type': 'application/json' },
|
| 210 |
-
body: JSON.stringify({ soul: soulContent, identity: identityContent }),
|
| 211 |
-
})
|
| 212 |
-
.then(() => { setPersonaDirty(false) })
|
| 213 |
-
.catch(() => {})
|
| 214 |
-
.finally(() => setPersonaSaving(false))
|
| 215 |
-
}, [soulContent, identityContent])
|
| 216 |
-
|
| 217 |
-
const generatePersona = useCallback(async () => {
|
| 218 |
-
if (!captureVrmScreenshot) return
|
| 219 |
-
const dataUrl = captureVrmScreenshot()
|
| 220 |
-
if (!dataUrl) return
|
| 221 |
-
setGenerating(true)
|
| 222 |
-
try {
|
| 223 |
-
// Save screenshot to server first
|
| 224 |
-
const saveRes = await fetch(`${FRIEND_API}/persona/screenshot`, {
|
| 225 |
-
method: 'POST',
|
| 226 |
-
headers: { 'Content-Type': 'application/json' },
|
| 227 |
-
body: JSON.stringify({ image: dataUrl }),
|
| 228 |
-
})
|
| 229 |
-
const saveData = await saveRes.json()
|
| 230 |
-
if (!saveData.ok) throw new Error(saveData.error || 'save screenshot failed')
|
| 231 |
-
// Generate persona from saved screenshot
|
| 232 |
-
const genRes = await fetch(`${FRIEND_API}/persona/generate`, {
|
| 233 |
-
method: 'POST',
|
| 234 |
-
headers: { 'Content-Type': 'application/json' },
|
| 235 |
-
})
|
| 236 |
-
const genData = await genRes.json()
|
| 237 |
-
if (genData.soul || genData.identity) {
|
| 238 |
-
if (genData.soul) setSoulContent(genData.soul)
|
| 239 |
-
if (genData.identity) setIdentityContent(genData.identity)
|
| 240 |
-
setPersonaDirty(true)
|
| 241 |
-
}
|
| 242 |
-
} catch { /* ignore */ }
|
| 243 |
-
setGenerating(false)
|
| 244 |
-
}, [captureVrmScreenshot])
|
| 245 |
-
|
| 246 |
const postVoiceSettings = (body: Record<string, string | undefined>) => {
|
| 247 |
return fetch(`${FRIEND_API}/voice`, {
|
| 248 |
method: 'POST',
|
|
@@ -286,7 +180,6 @@ export function SettingsPanel({
|
|
| 286 |
}, [])
|
| 287 |
|
| 288 |
const preview = (voiceId: string) => {
|
| 289 |
-
// Stop any current preview first
|
| 290 |
stopPreview()
|
| 291 |
setPreviewingId(voiceId)
|
| 292 |
fetch(`${FRIEND_API}/preview`, {
|
|
@@ -326,13 +219,13 @@ export function SettingsPanel({
|
|
| 326 |
|
| 327 |
{/* Tabs */}
|
| 328 |
<div style={tabBarStyle}>
|
| 329 |
-
{(['general', 'voice', 'model'
|
| 330 |
<button
|
| 331 |
key={tb}
|
| 332 |
onClick={() => setTab(tb)}
|
| 333 |
style={{ ...tabStyle, ...(tab === tb ? activeTabStyle : {}) }}
|
| 334 |
>
|
| 335 |
-
{{ general: t('常规', 'General'), voice: t('语音', 'Voice'), model: t('形象', 'Model')
|
| 336 |
</button>
|
| 337 |
))}
|
| 338 |
</div>
|
|
@@ -341,99 +234,13 @@ export function SettingsPanel({
|
|
| 341 |
<div style={contentStyle}>
|
| 342 |
{tab === 'general' && (
|
| 343 |
<div style={{ display: 'flex', flexDirection: 'column', gap: 16 }}>
|
| 344 |
-
<
|
| 345 |
-
<span style={{ fontSize: 14 }}>{t('语言', 'Language')}</span>
|
| 346 |
-
<div style={{ display: 'flex', gap: 4 }}>
|
| 347 |
-
{(['zh', 'en'] as const).map((l) => (
|
| 348 |
-
<button
|
| 349 |
-
key={l}
|
| 350 |
-
onClick={() => onLanguageChange(l)}
|
| 351 |
-
style={{
|
| 352 |
-
...smallBtnStyle,
|
| 353 |
-
background: language === l ? 'rgba(100, 160, 255, 0.4)' : 'rgba(255, 255, 255, 0.08)',
|
| 354 |
-
borderColor: language === l ? 'rgba(100, 160, 255, 0.6)' : 'rgba(255, 255, 255, 0.15)',
|
| 355 |
-
}}
|
| 356 |
-
>
|
| 357 |
-
{l === 'zh' ? '中文' : 'English'}
|
| 358 |
-
</button>
|
| 359 |
-
))}
|
| 360 |
-
</div>
|
| 361 |
-
</div>
|
| 362 |
<ToggleRow label={t('显示字幕', 'Subtitles')} value={showText} onChange={onShowTextChange} />
|
| 363 |
<ToggleRow label={t('语音播报', 'TTS')} value={ttsEnabled} onChange={onTtsEnabledChange} />
|
| 364 |
-
<
|
| 365 |
-
|
| 366 |
-
|
| 367 |
-
<input
|
| 368 |
-
type="range"
|
| 369 |
-
min={0}
|
| 370 |
-
max={100}
|
| 371 |
-
value={Math.round(volume * 100)}
|
| 372 |
-
onChange={(e) => onVolumeChange(Number(e.target.value) / 100)}
|
| 373 |
-
style={{ width: 100, accentColor: 'rgba(100, 160, 255, 0.8)' }}
|
| 374 |
-
/>
|
| 375 |
-
<span style={{ fontSize: 12, color: 'rgba(255,255,255,0.5)', width: 28, textAlign: 'right' }}>{Math.round(volume * 100)}</span>
|
| 376 |
-
</div>
|
| 377 |
-
</div>
|
| 378 |
-
<div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center' }}>
|
| 379 |
-
<span style={{ fontSize: 14 }}>{t('视线跟随', 'Eye Tracking')}</span>
|
| 380 |
-
<div style={{ display: 'flex', gap: 4 }}>
|
| 381 |
-
{(['mouse', 'camera'] as const).map((m) => (
|
| 382 |
-
<button
|
| 383 |
-
key={m}
|
| 384 |
-
onClick={() => onTrackingChange(m)}
|
| 385 |
-
style={{
|
| 386 |
-
...smallBtnStyle,
|
| 387 |
-
background: tracking === m ? 'rgba(100, 160, 255, 0.4)' : 'rgba(255, 255, 255, 0.08)',
|
| 388 |
-
borderColor: tracking === m ? 'rgba(100, 160, 255, 0.6)' : 'rgba(255, 255, 255, 0.15)',
|
| 389 |
-
}}
|
| 390 |
-
>
|
| 391 |
-
{m === 'mouse' ? t('鼠标', 'Mouse') : t('镜头', 'Camera')}
|
| 392 |
-
</button>
|
| 393 |
-
))}
|
| 394 |
-
</div>
|
| 395 |
-
</div>
|
| 396 |
-
<div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center' }}>
|
| 397 |
-
<span style={{ fontSize: 14 }}>{t('UI位置', 'UI Position')}</span>
|
| 398 |
-
<div style={{ display: 'flex', gap: 4 }}>
|
| 399 |
-
{(['left', 'right'] as const).map((a) => (
|
| 400 |
-
<button
|
| 401 |
-
key={a}
|
| 402 |
-
onClick={() => onUiAlignChange(a)}
|
| 403 |
-
style={{
|
| 404 |
-
...smallBtnStyle,
|
| 405 |
-
background: uiAlign === a ? 'rgba(100, 160, 255, 0.4)' : 'rgba(255, 255, 255, 0.08)',
|
| 406 |
-
borderColor: uiAlign === a ? 'rgba(100, 160, 255, 0.6)' : 'rgba(255, 255, 255, 0.15)',
|
| 407 |
-
}}
|
| 408 |
-
>
|
| 409 |
-
{a === 'left' ? t('靠左', 'Left') : t('靠右', 'Right')}
|
| 410 |
-
</button>
|
| 411 |
-
))}
|
| 412 |
-
</div>
|
| 413 |
-
</div>
|
| 414 |
<ToggleRow label={t('隐藏UI', 'Hide UI')} value={hideUI} onChange={onHideUIChange} />
|
| 415 |
-
<ToggleRow label={t('隐藏心情条', 'Hide Mood Bar')} value={hideMood} onChange={onHideMoodChange} />
|
| 416 |
-
<ToggleRow label={t('屏幕观察', 'Screen Observe')} value={screenObserve} onChange={onScreenObserveChange} />
|
| 417 |
-
{screenObserve && (
|
| 418 |
-
<>
|
| 419 |
-
<div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center', marginTop: -8 }}>
|
| 420 |
-
<span style={{ fontSize: 14 }}>{t('观察间隔', 'Interval')}</span>
|
| 421 |
-
<div style={{ display: 'flex', alignItems: 'center', gap: 8 }}>
|
| 422 |
-
<input
|
| 423 |
-
type="range"
|
| 424 |
-
min={15}
|
| 425 |
-
max={300}
|
| 426 |
-
step={15}
|
| 427 |
-
value={screenObserveInterval}
|
| 428 |
-
onChange={(e) => onScreenObserveIntervalChange(Number(e.target.value))}
|
| 429 |
-
style={{ width: 100, accentColor: 'rgba(100, 160, 255, 0.8)' }}
|
| 430 |
-
/>
|
| 431 |
-
<span style={{ fontSize: 12, color: 'rgba(255,255,255,0.5)', width: 36, textAlign: 'right' }}>{screenObserveInterval}s</span>
|
| 432 |
-
</div>
|
| 433 |
-
</div>
|
| 434 |
-
|
| 435 |
-
</>
|
| 436 |
-
)}
|
| 437 |
</div>
|
| 438 |
)}
|
| 439 |
|
|
@@ -513,11 +320,6 @@ export function SettingsPanel({
|
|
| 513 |
</button>
|
| 514 |
))}
|
| 515 |
</div>
|
| 516 |
-
<div style={{ fontSize: 11, color: 'rgba(255,255,255,0.35)', marginTop: 4 }}>
|
| 517 |
-
{sttProvider === 'browser'
|
| 518 |
-
? t('使用浏览器内置语音识别(Web Speech API)', 'Use browser built-in speech recognition (Web Speech API)')
|
| 519 |
-
: t('使用服务端语音识别,需要登录对应服务', 'Use server-side speech recognition, requires corresponding service login')}
|
| 520 |
-
</div>
|
| 521 |
</div>
|
| 522 |
|
| 523 |
<div style={{ marginTop: 8 }}>
|
|
@@ -555,7 +357,6 @@ export function SettingsPanel({
|
|
| 555 |
cursor: previewingId !== null ? 'default' : 'pointer',
|
| 556 |
flexShrink: 0,
|
| 557 |
opacity: previewingId !== null && previewingId !== v.id ? 0.3 : 0.7,
|
| 558 |
-
transition: 'background 0.15s',
|
| 559 |
}}
|
| 560 |
title={t('试听', 'Preview')}
|
| 561 |
>
|
|
@@ -581,268 +382,97 @@ export function SettingsPanel({
|
|
| 581 |
<option key={m} value={m}>{m.replace(/^\//, '')}</option>
|
| 582 |
))}
|
| 583 |
</select>
|
| 584 |
-
|
| 585 |
-
{models.length > 0 && (
|
| 586 |
-
<div style={{ marginTop: 12 }}>
|
| 587 |
-
<div style={labelStyle}>{t('自定义VRM模型', 'Custom VRM Models')}</div>
|
| 588 |
-
<select
|
| 589 |
-
value={!BUILTIN_MODELS.includes(currentModel) ? currentModel : ''}
|
| 590 |
-
onChange={(e) => { onModelChange(e.target.value); saveModelPath(e.target.value) }}
|
| 591 |
-
style={selectStyle}
|
| 592 |
-
>
|
| 593 |
-
{BUILTIN_MODELS.includes(currentModel) && <option value="" disabled>{t('未选择', 'Not selected')}</option>}
|
| 594 |
-
{models.map((m) => (
|
| 595 |
-
<option key={m} value={m}>{decodeURIComponent(m.split('/').pop() || m)}</option>
|
| 596 |
-
))}
|
| 597 |
-
</select>
|
| 598 |
-
</div>
|
| 599 |
-
)}
|
| 600 |
-
|
| 601 |
-
<div style={{ marginTop: 12 }}>
|
| 602 |
-
<div style={labelStyle}>{t('导入自定义模型', 'Import Custom Model')}</div>
|
| 603 |
-
<button
|
| 604 |
-
onClick={async () => {
|
| 605 |
-
const file = await pickFile('.vrm,.glb')
|
| 606 |
-
if (!file) return
|
| 607 |
-
try {
|
| 608 |
-
const formData = new FormData()
|
| 609 |
-
formData.append('file', file)
|
| 610 |
-
const res = await fetch(`${FRIEND_API}/model/import`, {
|
| 611 |
-
method: 'POST',
|
| 612 |
-
body: formData,
|
| 613 |
-
})
|
| 614 |
-
const data = await res.json()
|
| 615 |
-
if (data.url) {
|
| 616 |
-
fetch(`${FRIEND_API}/model/list`)
|
| 617 |
-
.then((r) => r.json())
|
| 618 |
-
.then((d) => { if (d.models) setModels(d.models) })
|
| 619 |
-
.catch(() => {})
|
| 620 |
-
onModelChange(data.url)
|
| 621 |
-
saveModelPath(data.url)
|
| 622 |
-
}
|
| 623 |
-
} catch (err) {
|
| 624 |
-
console.warn('Import model failed:', err)
|
| 625 |
-
}
|
| 626 |
-
}}
|
| 627 |
-
style={{ ...applyBtnStyle, width: '100%' }}
|
| 628 |
-
>
|
| 629 |
-
{t('浏览本地文件…', 'Browse local files…')}
|
| 630 |
-
</button>
|
| 631 |
-
<div style={{ fontSize: 11, color: 'rgba(255,255,255,0.35)', marginTop: 4 }}>
|
| 632 |
-
{t('选择本地 .vrm 文件,将保存到工作区 models 目录', 'Select a .vrm file to save to workspace models directory')}
|
| 633 |
-
</div>
|
| 634 |
-
</div>
|
| 635 |
</div>
|
| 636 |
)}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 637 |
|
| 638 |
-
|
| 639 |
-
|
| 640 |
-
|
| 641 |
-
|
| 642 |
-
|
| 643 |
-
|
| 644 |
-
|
| 645 |
-
|
| 646 |
-
|
| 647 |
-
|
| 648 |
-
|
| 649 |
-
|
| 650 |
-
|
| 651 |
-
|
| 652 |
-
|
| 653 |
-
|
| 654 |
-
|
| 655 |
-
|
| 656 |
-
|
| 657 |
-
|
| 658 |
-
|
| 659 |
-
|
| 660 |
-
onClick={generatePersona}
|
| 661 |
-
disabled={generating || !captureVrmScreenshot}
|
| 662 |
-
style={{
|
| 663 |
-
...applyBtnStyle,
|
| 664 |
-
flex: 1,
|
| 665 |
-
display: 'flex',
|
| 666 |
-
alignItems: 'center',
|
| 667 |
-
justifyContent: 'center',
|
| 668 |
-
gap: 4,
|
| 669 |
-
background: generating ? 'rgba(100, 160, 255, 0.3)' : 'rgba(160, 120, 255, 0.5)',
|
| 670 |
-
}}
|
| 671 |
-
title={t('根据当前模型截图自动生成人设', 'Auto-generate persona from model screenshot')}
|
| 672 |
-
>
|
| 673 |
-
{generating
|
| 674 |
-
? <Loader size={14} style={{ animation: 'spin 1s linear infinite' }} />
|
| 675 |
-
: <Sparkles size={14} />}
|
| 676 |
-
{generating ? t('生成中…', 'Generating…') : t('一键生成人设', 'Auto Generate')}
|
| 677 |
-
</button>
|
| 678 |
-
<button
|
| 679 |
-
onClick={savePersona}
|
| 680 |
-
disabled={!personaDirty || personaSaving}
|
| 681 |
-
style={{
|
| 682 |
-
...applyBtnStyle,
|
| 683 |
-
opacity: personaDirty ? 1 : 0.4,
|
| 684 |
-
}}
|
| 685 |
-
>
|
| 686 |
-
{personaSaving ? t('保存中…', 'Saving…') : t('保存', 'Save')}
|
| 687 |
-
</button>
|
| 688 |
-
</div>
|
| 689 |
-
<div style={{ fontSize: 11, color: 'rgba(255,255,255,0.3)', marginTop: 4 }}>
|
| 690 |
-
{t('关联工作区根目录的 IDENTITY.md 和 SOUL.md', 'Linked to IDENTITY.md and SOUL.md in workspace root')}
|
| 691 |
-
</div>
|
| 692 |
-
</div>
|
| 693 |
-
)}
|
| 694 |
-
|
| 695 |
-
{tab === 'dance' && (
|
| 696 |
-
<div style={sectionStyle}>
|
| 697 |
-
<div style={labelStyle}>{t('内置舞蹈', 'Built-in Dances')}</div>
|
| 698 |
-
<div style={{ display: 'flex', flexDirection: 'column', gap: 4 }}>
|
| 699 |
-
{Object.entries(dancePresets).map(([id, preset]) => (
|
| 700 |
-
<div
|
| 701 |
-
key={id}
|
| 702 |
-
onClick={() => onDanceChange(id)}
|
| 703 |
-
style={{
|
| 704 |
-
...modelBtnStyle,
|
| 705 |
-
display: 'flex',
|
| 706 |
-
alignItems: 'center',
|
| 707 |
-
justifyContent: 'space-between',
|
| 708 |
-
padding: '8px 12px',
|
| 709 |
-
fontSize: 13,
|
| 710 |
-
background: currentDance === id ? 'rgba(100, 160, 255, 0.4)' : 'rgba(255, 255, 255, 0.08)',
|
| 711 |
-
borderColor: currentDance === id ? 'rgba(100, 160, 255, 0.6)' : 'rgba(255, 255, 255, 0.15)',
|
| 712 |
-
}}
|
| 713 |
-
>
|
| 714 |
-
<div style={{ display: 'flex', alignItems: 'center', gap: 8 }}>
|
| 715 |
-
<Music size={14} style={{ opacity: 0.6 }} />
|
| 716 |
-
<span>{preset.label}</span>
|
| 717 |
-
</div>
|
| 718 |
-
{preset.bgm && (
|
| 719 |
-
<span style={{ fontSize: 11, color: 'rgba(255,255,255,0.35)' }}>{t('含BGM', 'w/ BGM')}</span>
|
| 720 |
-
)}
|
| 721 |
-
</div>
|
| 722 |
-
))}
|
| 723 |
-
</div>
|
| 724 |
-
|
| 725 |
-
{customDances.length > 0 && (
|
| 726 |
-
<div style={{ marginTop: 12 }}>
|
| 727 |
-
<div style={labelStyle}>{t('自定义舞蹈', 'Custom Dances')}</div>
|
| 728 |
-
<div style={{ display: 'flex', flexDirection: 'column', gap: 4 }}>
|
| 729 |
-
{customDances.map((dance) => (
|
| 730 |
-
<div
|
| 731 |
-
key={dance.id}
|
| 732 |
-
onClick={() => onDanceChange(`custom:${dance.id}`, {
|
| 733 |
-
label: dance.label,
|
| 734 |
-
type: 'vmd',
|
| 735 |
-
url: dance.vmdUrl,
|
| 736 |
-
bgm: dance.bgmUrl,
|
| 737 |
-
})}
|
| 738 |
-
style={{
|
| 739 |
-
...modelBtnStyle,
|
| 740 |
-
display: 'flex',
|
| 741 |
-
alignItems: 'center',
|
| 742 |
-
justifyContent: 'space-between',
|
| 743 |
-
padding: '8px 10px',
|
| 744 |
-
fontSize: 13,
|
| 745 |
-
background: currentDance === `custom:${dance.id}` ? 'rgba(100, 160, 255, 0.4)' : 'rgba(255, 255, 255, 0.08)',
|
| 746 |
-
borderColor: currentDance === `custom:${dance.id}` ? 'rgba(100, 160, 255, 0.6)' : 'rgba(255, 255, 255, 0.15)',
|
| 747 |
-
}}
|
| 748 |
-
>
|
| 749 |
-
<div style={{ display: 'flex', alignItems: 'center', gap: 8 }}>
|
| 750 |
-
<Music size={14} style={{ opacity: 0.6 }} />
|
| 751 |
-
<div>
|
| 752 |
-
<div>{dance.label}</div>
|
| 753 |
-
{dance.bgmUrl && (
|
| 754 |
-
<div style={{ fontSize: 11, color: 'rgba(255,255,255,0.35)', marginTop: 1 }}>{t('含BGM', 'w/ BGM')}</div>
|
| 755 |
-
)}
|
| 756 |
-
</div>
|
| 757 |
-
</div>
|
| 758 |
-
<div
|
| 759 |
-
onClick={(e) => {
|
| 760 |
-
e.stopPropagation()
|
| 761 |
-
fetch(`${FRIEND_API}/dance/delete`, {
|
| 762 |
-
method: 'POST',
|
| 763 |
-
headers: { 'Content-Type': 'application/json' },
|
| 764 |
-
body: JSON.stringify({ id: dance.id }),
|
| 765 |
-
})
|
| 766 |
-
.then(() => {
|
| 767 |
-
fetchCustomDances()
|
| 768 |
-
if (currentDance === `custom:${dance.id}`) onDanceChange('love')
|
| 769 |
-
})
|
| 770 |
-
.catch(() => {})
|
| 771 |
-
}}
|
| 772 |
-
style={{
|
| 773 |
-
width: 28,
|
| 774 |
-
height: 28,
|
| 775 |
-
borderRadius: 6,
|
| 776 |
-
display: 'flex',
|
| 777 |
-
alignItems: 'center',
|
| 778 |
-
justifyContent: 'center',
|
| 779 |
-
background: 'rgba(255, 255, 255, 0.08)',
|
| 780 |
-
cursor: 'pointer',
|
| 781 |
-
flexShrink: 0,
|
| 782 |
-
opacity: 0.5,
|
| 783 |
-
}}
|
| 784 |
-
title={t('删除', 'Delete')}
|
| 785 |
-
>
|
| 786 |
-
<Trash2 size={13} />
|
| 787 |
-
</div>
|
| 788 |
-
</div>
|
| 789 |
-
))}
|
| 790 |
-
</div>
|
| 791 |
-
</div>
|
| 792 |
-
)}
|
| 793 |
-
|
| 794 |
-
<div style={{ marginTop: 12 }}>
|
| 795 |
-
<div style={labelStyle}>{t('导入自定义舞蹈', 'Import Custom Dance')}</div>
|
| 796 |
-
<button
|
| 797 |
-
disabled={importingDance}
|
| 798 |
-
onClick={async () => {
|
| 799 |
-
setImportingDance(true)
|
| 800 |
-
try {
|
| 801 |
-
// Pick VMD file
|
| 802 |
-
const vmdFile = await pickFile('.vmd')
|
| 803 |
-
if (!vmdFile) { setImportingDance(false); return }
|
| 804 |
|
| 805 |
-
|
| 806 |
-
|
| 807 |
-
|
| 808 |
-
|
| 809 |
-
|
| 810 |
-
|
| 811 |
-
|
| 812 |
-
|
| 813 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 814 |
|
| 815 |
-
|
| 816 |
-
|
| 817 |
-
|
| 818 |
-
|
| 819 |
-
|
| 820 |
-
|
| 821 |
-
|
| 822 |
-
|
| 823 |
-
|
| 824 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 825 |
|
| 826 |
-
|
| 827 |
-
|
| 828 |
-
|
| 829 |
-
|
| 830 |
-
|
| 831 |
-
|
| 832 |
-
|
| 833 |
-
|
| 834 |
-
|
| 835 |
-
|
| 836 |
-
|
| 837 |
-
|
| 838 |
-
|
| 839 |
-
|
| 840 |
-
|
| 841 |
-
|
| 842 |
-
|
| 843 |
-
|
| 844 |
-
)}
|
| 845 |
-
</div>
|
| 846 |
</div>
|
| 847 |
</div>
|
| 848 |
)
|
|
@@ -1025,29 +655,3 @@ const selectStyle: React.CSSProperties = {
|
|
| 1025 |
outline: 'none',
|
| 1026 |
fontFamily: '"Segoe UI", "Microsoft YaHei", sans-serif',
|
| 1027 |
}
|
| 1028 |
-
|
| 1029 |
-
const textareaStyle: React.CSSProperties = {
|
| 1030 |
-
width: '100%',
|
| 1031 |
-
boxSizing: 'border-box',
|
| 1032 |
-
border: '1px solid rgba(255, 255, 255, 0.2)',
|
| 1033 |
-
borderRadius: 6,
|
| 1034 |
-
background: 'rgba(0, 0, 0, 0.3)',
|
| 1035 |
-
color: '#fff',
|
| 1036 |
-
fontSize: 12,
|
| 1037 |
-
lineHeight: '1.5',
|
| 1038 |
-
padding: '6px 8px',
|
| 1039 |
-
outline: 'none',
|
| 1040 |
-
fontFamily: '"Segoe UI", "Microsoft YaHei", sans-serif',
|
| 1041 |
-
resize: 'vertical',
|
| 1042 |
-
}
|
| 1043 |
-
|
| 1044 |
-
const applyBtnStyle: React.CSSProperties = {
|
| 1045 |
-
height: 32,
|
| 1046 |
-
padding: '0 12px',
|
| 1047 |
-
border: 'none',
|
| 1048 |
-
borderRadius: 6,
|
| 1049 |
-
background: 'rgba(100, 160, 255, 0.5)',
|
| 1050 |
-
color: '#fff',
|
| 1051 |
-
fontSize: 13,
|
| 1052 |
-
cursor: 'pointer',
|
| 1053 |
-
}
|
|
|
|
| 1 |
import { useState, useEffect, useRef, useCallback } from 'react'
|
| 2 |
+
import { X, Play, Loader } from 'lucide-react'
|
| 3 |
import { FRIEND_API } from '../api'
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4 |
|
| 5 |
type SttProvider = 'browser' | 'groq' | 'anthropic' | 'local' | 'doubao'
|
| 6 |
|
|
|
|
| 21 |
onVolumeChange: (v: number) => void
|
| 22 |
uiAlign: 'left' | 'right'
|
| 23 |
onUiAlignChange: (v: 'left' | 'right') => void
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 24 |
language: 'zh' | 'en'
|
| 25 |
onLanguageChange: (v: 'zh' | 'en') => void
|
|
|
|
|
|
|
| 26 |
sttProvider?: SttProvider
|
| 27 |
onSttProviderChange?: (v: SttProvider) => void
|
| 28 |
}
|
| 29 |
|
| 30 |
+
type Tab = 'general' | 'voice' | 'model'
|
| 31 |
|
| 32 |
const BUILTIN_MODELS = ['/friend/model1.vrm', '/friend/model2.vrm', '/friend/model3.vrm', '/friend/model4.vrm', '/friend/model5.vrm']
|
| 33 |
|
|
|
|
| 83 |
tracking, onTrackingChange,
|
| 84 |
volume, onVolumeChange,
|
| 85 |
uiAlign, onUiAlignChange,
|
|
|
|
|
|
|
|
|
|
|
|
|
| 86 |
language, onLanguageChange,
|
|
|
|
| 87 |
sttProvider = 'browser', onSttProviderChange,
|
| 88 |
}: SettingsPanelProps) {
|
| 89 |
const t = (zh: string, en: string) => language === 'en' ? en : zh
|
| 90 |
|
| 91 |
const [tab, setTab] = useState<Tab>('general')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 92 |
const [currentVoice, setCurrentVoice] = useState('')
|
| 93 |
const [currentProvider, setCurrentProvider] = useState<string>('edge')
|
| 94 |
const [qwenKey, setQwenKey] = useState('')
|
| 95 |
const [qwenModel, setQwenModel] = useState('qwen3-tts-flash')
|
| 96 |
const [previewingId, setPreviewingId] = useState<string | null>(null)
|
| 97 |
const audioRef = useRef<HTMLAudioElement | null>(null)
|
|
|
|
|
|
|
|
|
|
| 98 |
|
| 99 |
// Drag state
|
| 100 |
const [panelPos, setPanelPos] = useState<{ x: number; y: number }>({ x: 0, y: 0 })
|
|
|
|
| 120 |
window.addEventListener('mouseup', onUp)
|
| 121 |
}, [panelPos])
|
| 122 |
|
|
|
|
| 123 |
useEffect(() => {
|
| 124 |
if (visible) setPanelPos({ x: 0, y: 0 })
|
| 125 |
}, [visible])
|
| 126 |
|
| 127 |
useEffect(() => {
|
| 128 |
if (!visible) return
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 129 |
fetch(`${FRIEND_API}/voice`)
|
| 130 |
.then((r) => r.json())
|
| 131 |
.then((data) => {
|
|
|
|
| 137 |
.catch(() => {})
|
| 138 |
}, [visible])
|
| 139 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 140 |
const postVoiceSettings = (body: Record<string, string | undefined>) => {
|
| 141 |
return fetch(`${FRIEND_API}/voice`, {
|
| 142 |
method: 'POST',
|
|
|
|
| 180 |
}, [])
|
| 181 |
|
| 182 |
const preview = (voiceId: string) => {
|
|
|
|
| 183 |
stopPreview()
|
| 184 |
setPreviewingId(voiceId)
|
| 185 |
fetch(`${FRIEND_API}/preview`, {
|
|
|
|
| 219 |
|
| 220 |
{/* Tabs */}
|
| 221 |
<div style={tabBarStyle}>
|
| 222 |
+
{(['general', 'voice', 'model'] as const).map((tb) => (
|
| 223 |
<button
|
| 224 |
key={tb}
|
| 225 |
onClick={() => setTab(tb)}
|
| 226 |
style={{ ...tabStyle, ...(tab === tb ? activeTabStyle : {}) }}
|
| 227 |
>
|
| 228 |
+
{{ general: t('常规', 'General'), voice: t('语音', 'Voice'), model: t('形象', 'Model') }[tb]}
|
| 229 |
</button>
|
| 230 |
))}
|
| 231 |
</div>
|
|
|
|
| 234 |
<div style={contentStyle}>
|
| 235 |
{tab === 'general' && (
|
| 236 |
<div style={{ display: 'flex', flexDirection: 'column', gap: 16 }}>
|
| 237 |
+
<LangToggle language={language} onChange={onLanguageChange} t={t} />
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 238 |
<ToggleRow label={t('显示字幕', 'Subtitles')} value={showText} onChange={onShowTextChange} />
|
| 239 |
<ToggleRow label={t('语音播报', 'TTS')} value={ttsEnabled} onChange={onTtsEnabledChange} />
|
| 240 |
+
<VolumeControl volume={volume} onChange={onVolumeChange} t={t} />
|
| 241 |
+
<TrackingControl tracking={tracking} onChange={onTrackingChange} t={t} />
|
| 242 |
+
<UIAlignControl uiAlign={uiAlign} onChange={onUiAlignChange} t={t} />
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 243 |
<ToggleRow label={t('隐藏UI', 'Hide UI')} value={hideUI} onChange={onHideUIChange} />
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 244 |
</div>
|
| 245 |
)}
|
| 246 |
|
|
|
|
| 320 |
</button>
|
| 321 |
))}
|
| 322 |
</div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 323 |
</div>
|
| 324 |
|
| 325 |
<div style={{ marginTop: 8 }}>
|
|
|
|
| 357 |
cursor: previewingId !== null ? 'default' : 'pointer',
|
| 358 |
flexShrink: 0,
|
| 359 |
opacity: previewingId !== null && previewingId !== v.id ? 0.3 : 0.7,
|
|
|
|
| 360 |
}}
|
| 361 |
title={t('试听', 'Preview')}
|
| 362 |
>
|
|
|
|
| 382 |
<option key={m} value={m}>{m.replace(/^\//, '')}</option>
|
| 383 |
))}
|
| 384 |
</select>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 385 |
</div>
|
| 386 |
)}
|
| 387 |
+
</div>
|
| 388 |
+
</div>
|
| 389 |
+
</div>
|
| 390 |
+
)
|
| 391 |
+
}
|
| 392 |
|
| 393 |
+
function LangToggle({ language, onChange, t }: { language: 'zh' | 'en'; onChange: (v: 'zh' | 'en') => void; t: (zh: string, en: string) => string }) {
|
| 394 |
+
return (
|
| 395 |
+
<div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center' }}>
|
| 396 |
+
<span style={{ fontSize: 14 }}>{t('语言', 'Language')}</span>
|
| 397 |
+
<div style={{ display: 'flex', gap: 4 }}>
|
| 398 |
+
{(['zh', 'en'] as const).map((l) => (
|
| 399 |
+
<button
|
| 400 |
+
key={l}
|
| 401 |
+
onClick={() => onChange(l)}
|
| 402 |
+
style={{
|
| 403 |
+
...smallBtnStyle,
|
| 404 |
+
background: language === l ? 'rgba(100, 160, 255, 0.4)' : 'rgba(255, 255, 255, 0.08)',
|
| 405 |
+
borderColor: language === l ? 'rgba(100, 160, 255, 0.6)' : 'rgba(255, 255, 255, 0.15)',
|
| 406 |
+
}}
|
| 407 |
+
>
|
| 408 |
+
{l === 'zh' ? '中文' : 'English'}
|
| 409 |
+
</button>
|
| 410 |
+
))}
|
| 411 |
+
</div>
|
| 412 |
+
</div>
|
| 413 |
+
)
|
| 414 |
+
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 415 |
|
| 416 |
+
function VolumeControl({ volume, onChange, t }: { volume: number; onChange: (v: number) => void; t: (zh: string, en: string) => string }) {
|
| 417 |
+
return (
|
| 418 |
+
<div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center' }}>
|
| 419 |
+
<span style={{ fontSize: 14 }}>{t('音量', 'Volume')}</span>
|
| 420 |
+
<div style={{ display: 'flex', alignItems: 'center', gap: 8 }}>
|
| 421 |
+
<input
|
| 422 |
+
type="range"
|
| 423 |
+
min={0}
|
| 424 |
+
max={100}
|
| 425 |
+
value={Math.round(volume * 100)}
|
| 426 |
+
onChange={(e) => onChange(Number(e.target.value) / 100)}
|
| 427 |
+
style={{ width: 100, accentColor: 'rgba(100, 160, 255, 0.8)' }}
|
| 428 |
+
/>
|
| 429 |
+
<span style={{ fontSize: 12, color: 'rgba(255,255,255,0.5)', width: 28, textAlign: 'right' }}>{Math.round(volume * 100)}</span>
|
| 430 |
+
</div>
|
| 431 |
+
</div>
|
| 432 |
+
)
|
| 433 |
+
}
|
| 434 |
|
| 435 |
+
function TrackingControl({ tracking, onChange, t }: { tracking: 'mouse' | 'camera'; onChange: (v: 'mouse' | 'camera') => void; t: (zh: string, en: string) => string }) {
|
| 436 |
+
return (
|
| 437 |
+
<div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center' }}>
|
| 438 |
+
<span style={{ fontSize: 14 }}>{t('视线跟随', 'Eye Tracking')}</span>
|
| 439 |
+
<div style={{ display: 'flex', gap: 4 }}>
|
| 440 |
+
{(['mouse', 'camera'] as const).map((m) => (
|
| 441 |
+
<button
|
| 442 |
+
key={m}
|
| 443 |
+
onClick={() => onChange(m)}
|
| 444 |
+
style={{
|
| 445 |
+
...smallBtnStyle,
|
| 446 |
+
background: tracking === m ? 'rgba(100, 160, 255, 0.4)' : 'rgba(255, 255, 255, 0.08)',
|
| 447 |
+
borderColor: tracking === m ? 'rgba(100, 160, 255, 0.6)' : 'rgba(255, 255, 255, 0.15)',
|
| 448 |
+
}}
|
| 449 |
+
>
|
| 450 |
+
{m === 'mouse' ? t('鼠标', 'Mouse') : t('镜头', 'Camera')}
|
| 451 |
+
</button>
|
| 452 |
+
))}
|
| 453 |
+
</div>
|
| 454 |
+
</div>
|
| 455 |
+
)
|
| 456 |
+
}
|
| 457 |
|
| 458 |
+
function UIAlignControl({ uiAlign, onChange, t }: { uiAlign: 'left' | 'right'; onChange: (v: 'left' | 'right') => void; t: (zh: string, en: string) => string }) {
|
| 459 |
+
return (
|
| 460 |
+
<div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center' }}>
|
| 461 |
+
<span style={{ fontSize: 14 }}>{t('UI位置', 'UI Position')}</span>
|
| 462 |
+
<div style={{ display: 'flex', gap: 4 }}>
|
| 463 |
+
{(['left', 'right'] as const).map((a) => (
|
| 464 |
+
<button
|
| 465 |
+
key={a}
|
| 466 |
+
onClick={() => onChange(a)}
|
| 467 |
+
style={{
|
| 468 |
+
...smallBtnStyle,
|
| 469 |
+
background: uiAlign === a ? 'rgba(100, 160, 255, 0.4)' : 'rgba(255, 255, 255, 0.08)',
|
| 470 |
+
borderColor: uiAlign === a ? 'rgba(100, 160, 255, 0.6)' : 'rgba(255, 255, 255, 0.15)',
|
| 471 |
+
}}
|
| 472 |
+
>
|
| 473 |
+
{a === 'left' ? t('靠左', 'Left') : t('靠右', 'Right')}
|
| 474 |
+
</button>
|
| 475 |
+
))}
|
|
|
|
|
|
|
| 476 |
</div>
|
| 477 |
</div>
|
| 478 |
)
|
|
|
|
| 655 |
outline: 'none',
|
| 656 |
fontFamily: '"Segoe UI", "Microsoft YaHei", sans-serif',
|
| 657 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
src/components/friend/frontend/components/TextBubble.tsx
CHANGED
|
@@ -12,6 +12,7 @@ interface VrmMessage {
|
|
| 12 |
audioUrl?: string
|
| 13 |
audioIndex?: number
|
| 14 |
imageUrl?: string
|
|
|
|
| 15 |
sendFirstTts?: boolean
|
| 16 |
appendText?: boolean
|
| 17 |
replyDone?: boolean
|
|
|
|
| 12 |
audioUrl?: string
|
| 13 |
audioIndex?: number
|
| 14 |
imageUrl?: string
|
| 15 |
+
action?: string
|
| 16 |
sendFirstTts?: boolean
|
| 17 |
appendText?: boolean
|
| 18 |
replyDone?: boolean
|
src/components/friend/frontend/components/VRMScene.tsx
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
import { useEffect, useRef, useImperativeHandle, forwardRef
|
| 2 |
import * as THREE from 'three'
|
| 3 |
import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js'
|
| 4 |
import { VRMLoaderPlugin, VRMUtils } from '@pixiv/three-vrm'
|
|
@@ -8,13 +8,9 @@ import { EmoteController } from '../emote'
|
|
| 8 |
import { LipSync } from '../lip-sync'
|
| 9 |
import { MotionController } from '../motion-controller'
|
| 10 |
|
| 11 |
-
export type TouchRegion = 'head' | 'arm' | 'leg' | 'chest' | 'belly' | 'buttocks'
|
| 12 |
-
|
| 13 |
interface VRMSceneProps {
|
| 14 |
modelPath: string
|
| 15 |
idleAnimationPath?: string
|
| 16 |
-
onTouch?: (region: TouchRegion) => void
|
| 17 |
-
onModelLoaded?: () => void
|
| 18 |
}
|
| 19 |
|
| 20 |
export type TrackingMode = 'mouse' | 'camera'
|
|
@@ -25,13 +21,8 @@ export interface VRMSceneHandle {
|
|
| 25 |
resetCamera: () => void
|
| 26 |
setTrackingMode: (mode: TrackingMode) => void
|
| 27 |
playAction: (name: string, hold?: boolean) => void
|
| 28 |
-
captureScreenshot: () => string | null
|
| 29 |
panCamera: (dx: number, dy: number) => void
|
| 30 |
rotateCamera: (dx: number, dy: number) => void
|
| 31 |
-
playDance: (nameOrPreset: string | import('../motion-controller').DancePreset) => void
|
| 32 |
-
stopDance: () => void
|
| 33 |
-
isDancing: () => boolean
|
| 34 |
-
setBgmVolume: (v: number) => void
|
| 35 |
/** Unified reset: camera + resetToIdle + expressions to zero */
|
| 36 |
reset: () => void
|
| 37 |
}
|
|
@@ -42,6 +33,9 @@ interface BlinkState {
|
|
| 42 |
blinkProgress: number
|
| 43 |
timeSinceLastBlink: number
|
| 44 |
nextBlinkTime: number
|
|
|
|
|
|
|
|
|
|
| 45 |
}
|
| 46 |
|
| 47 |
function createBlinkState(): BlinkState {
|
|
@@ -49,7 +43,10 @@ function createBlinkState(): BlinkState {
|
|
| 49 |
isBlinking: false,
|
| 50 |
blinkProgress: 0,
|
| 51 |
timeSinceLastBlink: 0,
|
| 52 |
-
nextBlinkTime: Math.random() *
|
|
|
|
|
|
|
|
|
|
| 53 |
}
|
| 54 |
}
|
| 55 |
|
|
@@ -61,32 +58,37 @@ function updateBlink(vrm: VRM, delta: number, state: BlinkState) {
|
|
| 61 |
if (!state.isBlinking && state.timeSinceLastBlink >= state.nextBlinkTime) {
|
| 62 |
state.isBlinking = true
|
| 63 |
state.blinkProgress = 0
|
|
|
|
|
|
|
|
|
|
|
|
|
| 64 |
}
|
| 65 |
|
| 66 |
if (state.isBlinking) {
|
| 67 |
-
|
| 68 |
-
state.blinkProgress += delta / BLINK_DURATION
|
| 69 |
const blinkValue = Math.sin(Math.PI * state.blinkProgress)
|
| 70 |
vrm.expressionManager.setValue('blink', blinkValue)
|
| 71 |
|
| 72 |
if (state.blinkProgress >= 1) {
|
| 73 |
-
state.
|
| 74 |
-
state.
|
| 75 |
-
|
| 76 |
-
state.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 77 |
}
|
| 78 |
}
|
| 79 |
}
|
| 80 |
|
| 81 |
// ── Relaxed hand pose ─────────────────────────────────────────────────────────
|
| 82 |
-
// The idle_loop.vrma often has no finger tracks, so fingers stay in the stiff
|
| 83 |
-
// T-pose. This must be applied EVERY FRAME after mixer.update() because the
|
| 84 |
-
// AnimationMixer resets bones that have no tracks back to their rest rotation.
|
| 85 |
-
//
|
| 86 |
-
// VRM normalized bones use Z-axis for finger curl (spread is Y-axis).
|
| 87 |
-
// Left hand curls positive Z, right hand curls negative Z.
|
| 88 |
-
// We also add slight spread (Y-axis) variation per finger for a natural look,
|
| 89 |
-
// and subtle per-frame micro-movement to avoid a "frozen" appearance.
|
| 90 |
|
| 91 |
interface HandPoseCache {
|
| 92 |
bones: { bone: THREE.Object3D; z: number; y: number }[]
|
|
@@ -101,7 +103,6 @@ function buildHandPoseCache(vrm: VRM): HandPoseCache {
|
|
| 101 |
const segments = ['Proximal', 'Intermediate', 'Distal'] as const
|
| 102 |
const sides = ['left', 'right'] as const
|
| 103 |
|
| 104 |
-
// Base curl values — outer fingers curl more for a natural resting hand
|
| 105 |
const curlMap: Record<string, [number, number, number]> = {
|
| 106 |
Thumb: [0.25, 0.15, 0.10],
|
| 107 |
Index: [0.20, 0.30, 0.20],
|
|
@@ -110,7 +111,6 @@ function buildHandPoseCache(vrm: VRM): HandPoseCache {
|
|
| 110 |
Little: [0.35, 0.45, 0.30],
|
| 111 |
}
|
| 112 |
|
| 113 |
-
// Slight spread (Y-axis) to fan fingers apart naturally
|
| 114 |
const spreadMap: Record<string, number> = {
|
| 115 |
Thumb: 0.15,
|
| 116 |
Index: 0.04,
|
|
@@ -132,7 +132,6 @@ function buildHandPoseCache(vrm: VRM): HandPoseCache {
|
|
| 132 |
if (!bone) continue
|
| 133 |
|
| 134 |
const z = sign * curls[s]
|
| 135 |
-
// Only apply spread on the proximal segment
|
| 136 |
const y = s === 0 ? sign * spread : 0
|
| 137 |
|
| 138 |
bones.push({ bone, z, y })
|
|
@@ -144,32 +143,23 @@ function buildHandPoseCache(vrm: VRM): HandPoseCache {
|
|
| 144 |
}
|
| 145 |
|
| 146 |
function applyRelaxedHandPose(cache: HandPoseCache, time: number) {
|
|
|
|
|
|
|
|
|
|
|
|
|
| 147 |
for (const { bone, z, y } of cache.bones) {
|
| 148 |
-
// Subtle micro-movement: ±0.02 rad oscillation at slightly different
|
| 149 |
-
// frequencies per bone (seeded by the base z value) to avoid uniformity
|
| 150 |
const freq = 0.3 + Math.abs(z) * 2
|
| 151 |
const micro = Math.sin(time * freq + z * 50) * 0.02
|
| 152 |
-
bone.rotation.z = z + micro
|
| 153 |
if (y !== 0) bone.rotation.y = y
|
| 154 |
}
|
| 155 |
}
|
| 156 |
|
| 157 |
-
|
| 158 |
export const VRMScene = forwardRef<VRMSceneHandle, VRMSceneProps>(function VRMScene({
|
| 159 |
modelPath,
|
| 160 |
idleAnimationPath = '/friend/idle_loop.vrma',
|
| 161 |
-
onTouch,
|
| 162 |
-
onModelLoaded,
|
| 163 |
}, ref) {
|
| 164 |
const canvasRef = useRef<HTMLCanvasElement>(null)
|
| 165 |
-
const [ripples, setRipples] = useState<{ id: number; x: number; y: number; confirmed: boolean }[]>([])
|
| 166 |
-
const rippleIdRef = useRef(0)
|
| 167 |
-
|
| 168 |
-
const spawnRipple = useCallback((x: number, y: number, confirmed: boolean) => {
|
| 169 |
-
const id = ++rippleIdRef.current
|
| 170 |
-
setRipples(prev => [...prev, { id, x, y, confirmed }])
|
| 171 |
-
setTimeout(() => setRipples(prev => prev.filter(r => r.id !== id)), confirmed ? 700 : 500)
|
| 172 |
-
}, [])
|
| 173 |
|
| 174 |
const emoteRef = useRef<EmoteController | null>(null)
|
| 175 |
const resetCameraRef = useRef<(() => void) | null>(null)
|
|
@@ -178,10 +168,6 @@ export const VRMScene = forwardRef<VRMSceneHandle, VRMSceneProps>(function VRMSc
|
|
| 178 |
const panCameraRef = useRef<((dx: number, dy: number) => void) | null>(null)
|
| 179 |
const rotateCameraRef = useRef<((dx: number, dy: number) => void) | null>(null)
|
| 180 |
const lipSyncRef = useRef<LipSync>(LipSync.getInstance())
|
| 181 |
-
const onTouchRef = useRef(onTouch)
|
| 182 |
-
onTouchRef.current = onTouch
|
| 183 |
-
const onModelLoadedRef = useRef(onModelLoaded)
|
| 184 |
-
onModelLoadedRef.current = onModelLoaded
|
| 185 |
|
| 186 |
useImperativeHandle(ref, () => ({
|
| 187 |
setEmotion(emotion: string, intensity?: number) {
|
|
@@ -199,27 +185,12 @@ export const VRMScene = forwardRef<VRMSceneHandle, VRMSceneProps>(function VRMSc
|
|
| 199 |
playAction(name: string, hold?: boolean) {
|
| 200 |
motionRef.current?.playAction(name, hold)
|
| 201 |
},
|
| 202 |
-
captureScreenshot() {
|
| 203 |
-
return canvasRef.current?.toDataURL('image/png') ?? null
|
| 204 |
-
},
|
| 205 |
panCamera(dx: number, dy: number) {
|
| 206 |
panCameraRef.current?.(dx, dy)
|
| 207 |
},
|
| 208 |
rotateCamera(dx: number, dy: number) {
|
| 209 |
rotateCameraRef.current?.(dx, dy)
|
| 210 |
},
|
| 211 |
-
playDance(nameOrPreset: string | import('../motion-controller').DancePreset) {
|
| 212 |
-
motionRef.current?.playDance(nameOrPreset)
|
| 213 |
-
},
|
| 214 |
-
stopDance() {
|
| 215 |
-
motionRef.current?.resetToIdle()
|
| 216 |
-
},
|
| 217 |
-
isDancing() {
|
| 218 |
-
return motionRef.current?.isDancing ?? false
|
| 219 |
-
},
|
| 220 |
-
setBgmVolume(v: number) {
|
| 221 |
-
motionRef.current?.setVolume(v)
|
| 222 |
-
},
|
| 223 |
reset() {
|
| 224 |
resetCameraRef.current?.()
|
| 225 |
motionRef.current?.resetToIdle()
|
|
@@ -253,12 +224,10 @@ export const VRMScene = forwardRef<VRMSceneHandle, VRMSceneProps>(function VRMSc
|
|
| 253 |
0.1,
|
| 254 |
100,
|
| 255 |
)
|
| 256 |
-
// Orbit state: camera orbits around the pivot point
|
| 257 |
-
// Will be recalculated after model loads (like airi)
|
| 258 |
const pivot = new THREE.Vector3(0, 0, 0)
|
| 259 |
let orbitRadius = 2.0
|
| 260 |
-
let orbitTheta = 0
|
| 261 |
-
let orbitPhi = Math.PI / 2
|
| 262 |
|
| 263 |
function updateCameraOrbit() {
|
| 264 |
camera.position.set(
|
|
@@ -292,6 +261,18 @@ export const VRMScene = forwardRef<VRMSceneHandle, VRMSceneProps>(function VRMSc
|
|
| 292 |
const saccades = new EyeSaccadeController()
|
| 293 |
const lookAtTarget = { x: 0, y: 0, z: -100 }
|
| 294 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 295 |
// ── Load VRM model, then load idle animation ─────────────────────────────
|
| 296 |
loader.load(
|
| 297 |
modelPath,
|
|
@@ -308,26 +289,24 @@ export const VRMScene = forwardRef<VRMSceneHandle, VRMSceneProps>(function VRMSc
|
|
| 308 |
obj.frustumCulled = false
|
| 309 |
})
|
| 310 |
|
| 311 |
-
// Add lookAt quaternion proxy (from airi — needed for lookAt to work)
|
| 312 |
if (loadedVrm.lookAt) {
|
| 313 |
const lookAtQuatProxy = new VRMLookAtQuaternionProxy(loadedVrm.lookAt)
|
| 314 |
lookAtQuatProxy.name = 'lookAtQuaternionProxy'
|
| 315 |
loadedVrm.scene.add(lookAtQuatProxy)
|
| 316 |
}
|
| 317 |
|
| 318 |
-
// Normalize VRM 0.x to match 1.0 convention, then face camera
|
| 319 |
VRMUtils.rotateVRM0(loadedVrm)
|
| 320 |
|
| 321 |
scene.add(loadedVrm.scene)
|
| 322 |
vrm = loadedVrm
|
| 323 |
|
| 324 |
-
// ── Compute camera from model bounds
|
| 325 |
const box = new THREE.Box3().setFromObject(loadedVrm.scene)
|
| 326 |
const modelSize = new THREE.Vector3()
|
| 327 |
const modelCenter = new THREE.Vector3()
|
| 328 |
box.getSize(modelSize)
|
| 329 |
box.getCenter(modelCenter)
|
| 330 |
-
modelCenter.y += modelSize.y / 3.2
|
| 331 |
|
| 332 |
const radians = (FOV / 2 * Math.PI) / 180
|
| 333 |
const offsetX = modelSize.x / 16
|
|
@@ -340,7 +319,6 @@ export const VRMScene = forwardRef<VRMSceneHandle, VRMSceneProps>(function VRMSc
|
|
| 340 |
orbitPhi = Math.PI / 2 - Math.atan2(offsetY, offsetZ)
|
| 341 |
updateCameraOrbit()
|
| 342 |
|
| 343 |
-
// Store initial state for reset
|
| 344 |
const initPivot = pivot.clone()
|
| 345 |
const initRadius = orbitRadius
|
| 346 |
const initTheta = orbitTheta
|
|
@@ -357,8 +335,8 @@ export const VRMScene = forwardRef<VRMSceneHandle, VRMSceneProps>(function VRMSc
|
|
| 357 |
const right = new THREE.Vector3()
|
| 358 |
const up = new THREE.Vector3()
|
| 359 |
camera.getWorldDirection(new THREE.Vector3())
|
| 360 |
-
right.setFromMatrixColumn(camera.matrixWorld, 0)
|
| 361 |
-
up.setFromMatrixColumn(camera.matrixWorld, 1)
|
| 362 |
pivot.addScaledVector(right, -dx * 0.003)
|
| 363 |
pivot.addScaledVector(up, dy * 0.003)
|
| 364 |
updateCameraOrbit()
|
|
@@ -374,54 +352,27 @@ export const VRMScene = forwardRef<VRMSceneHandle, VRMSceneProps>(function VRMSc
|
|
| 374 |
updateCameraOrbit()
|
| 375 |
}
|
| 376 |
|
| 377 |
-
// Build hand pose cache (applied every frame in animate loop)
|
| 378 |
handPose = buildHandPoseCache(loadedVrm)
|
| 379 |
|
| 380 |
-
// Initialize
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 381 |
emote = new EmoteController(loadedVrm)
|
| 382 |
emoteRef.current = emote
|
| 383 |
|
| 384 |
-
// ── Initialize MotionController ──────────────────────────────────────
|
| 385 |
motion = new MotionController(loadedVrm)
|
| 386 |
motionRef.current = motion
|
| 387 |
|
| 388 |
-
// Dance camera: auto-fit from hips position, centered on upper body
|
| 389 |
-
const danceRadius = (modelSize.y / 1.6) / Math.tan((FOV / 2 * Math.PI) / 180)
|
| 390 |
-
|
| 391 |
-
motion.onDanceStart = () => {
|
| 392 |
-
const hipsNode = loadedVrm.humanoid?.getNormalizedBoneNode('hips')
|
| 393 |
-
if (hipsNode) {
|
| 394 |
-
const hipsWorld = new THREE.Vector3()
|
| 395 |
-
hipsNode.getWorldPosition(hipsWorld)
|
| 396 |
-
// Pivot at hips height (upper body center)
|
| 397 |
-
pivot.set(hipsWorld.x, hipsWorld.y, hipsWorld.z)
|
| 398 |
-
} else {
|
| 399 |
-
pivot.copy(modelCenter)
|
| 400 |
-
}
|
| 401 |
-
orbitRadius = danceRadius
|
| 402 |
-
orbitTheta = 0
|
| 403 |
-
orbitPhi = Math.PI / 2
|
| 404 |
-
updateCameraOrbit()
|
| 405 |
-
}
|
| 406 |
-
motion.onDanceStop = () => {
|
| 407 |
-
pivot.copy(initPivot)
|
| 408 |
-
orbitRadius = initRadius
|
| 409 |
-
orbitTheta = initTheta
|
| 410 |
-
orbitPhi = initPhi
|
| 411 |
-
updateCameraOrbit()
|
| 412 |
-
}
|
| 413 |
-
|
| 414 |
-
// Load idle animation (non-blocking for fast startup)
|
| 415 |
motion.loadIdle(idleAnimationPath).catch((err) =>
|
| 416 |
console.warn('Failed to load idle animation:', err),
|
| 417 |
)
|
| 418 |
|
| 419 |
-
// Reset spring bones after everything is set up
|
| 420 |
loadedVrm.springBoneManager?.reset()
|
| 421 |
-
|
| 422 |
-
// Notify parent that model is ready (for auto-screenshot etc.)
|
| 423 |
-
// Delay slightly so the first frame is rendered
|
| 424 |
-
setTimeout(() => onModelLoadedRef.current?.(), 500)
|
| 425 |
},
|
| 426 |
() => {},
|
| 427 |
(err) => {
|
|
@@ -431,7 +382,6 @@ export const VRMScene = forwardRef<VRMSceneHandle, VRMSceneProps>(function VRMSc
|
|
| 431 |
|
| 432 |
// ── Mouse tracking ────────────────────────────────────────────────────────
|
| 433 |
const mouse = new THREE.Vector2(0, 0)
|
| 434 |
-
|
| 435 |
const _raycaster = new THREE.Raycaster()
|
| 436 |
const _mouseVec = new THREE.Vector2()
|
| 437 |
|
|
@@ -441,7 +391,6 @@ export const VRMScene = forwardRef<VRMSceneHandle, VRMSceneProps>(function VRMSc
|
|
| 441 |
|
| 442 |
if (trackingModeRef.current !== 'mouse') return
|
| 443 |
|
| 444 |
-
// Compute lookAt target like airi's lookAtMouse
|
| 445 |
_mouseVec.set(mouse.x, mouse.y)
|
| 446 |
_raycaster.setFromCamera(_mouseVec, camera)
|
| 447 |
const camDir = new THREE.Vector3()
|
|
@@ -461,7 +410,6 @@ export const VRMScene = forwardRef<VRMSceneHandle, VRMSceneProps>(function VRMSc
|
|
| 461 |
}
|
| 462 |
}
|
| 463 |
}
|
| 464 |
-
// Listen on both window and document to handle transparent window cases
|
| 465 |
window.addEventListener('mousemove', onMouseMove)
|
| 466 |
document.addEventListener('mousemove', onMouseMove)
|
| 467 |
|
|
@@ -481,92 +429,15 @@ export const VRMScene = forwardRef<VRMSceneHandle, VRMSceneProps>(function VRMSc
|
|
| 481 |
}
|
| 482 |
canvas.addEventListener('wheel', onWheel, { passive: false })
|
| 483 |
|
| 484 |
-
// ──
|
| 485 |
-
|
| 486 |
-
// Middle drag: dolly (zoom)
|
| 487 |
-
// Right drag: rotate around model
|
| 488 |
-
let dragMode: 'rotate' | 'dolly' | 'pan' | null = null
|
| 489 |
let prevX = 0
|
| 490 |
let prevY = 0
|
| 491 |
const ROTATE_SPEED = 0.005
|
| 492 |
-
const PAN_SPEED = 0.003
|
| 493 |
const DOLLY_SPEED = 0.01
|
| 494 |
|
| 495 |
-
// ── Touch interaction: detect body region from raycast hit ────────────
|
| 496 |
-
const touchRaycaster = new THREE.Raycaster()
|
| 497 |
-
const touchMouseVec = new THREE.Vector2()
|
| 498 |
-
let lastTapTime = 0
|
| 499 |
-
let lastTapRegion: TouchRegion | null = null
|
| 500 |
-
let lastTouchFireTime = 0
|
| 501 |
-
const DOUBLE_TAP_WINDOW = 500 // ms
|
| 502 |
-
const TOUCH_COOLDOWN = 5_000 // ms
|
| 503 |
-
|
| 504 |
-
// Bone-to-region mapping for proximity-based touch detection
|
| 505 |
-
const boneRegionMap: [string, TouchRegion][] = [
|
| 506 |
-
// Head
|
| 507 |
-
['head', 'head'], ['neck', 'head'],
|
| 508 |
-
['leftEye', 'head'], ['rightEye', 'head'], ['jaw', 'head'],
|
| 509 |
-
// Arms
|
| 510 |
-
['leftShoulder', 'arm'], ['leftUpperArm', 'arm'], ['leftLowerArm', 'arm'], ['leftHand', 'arm'],
|
| 511 |
-
['rightShoulder', 'arm'], ['rightUpperArm', 'arm'], ['rightLowerArm', 'arm'], ['rightHand', 'arm'],
|
| 512 |
-
// Legs
|
| 513 |
-
['leftUpperLeg', 'leg'], ['leftLowerLeg', 'leg'], ['leftFoot', 'leg'], ['leftToes', 'leg'],
|
| 514 |
-
['rightUpperLeg', 'leg'], ['rightLowerLeg', 'leg'], ['rightFoot', 'leg'], ['rightToes', 'leg'],
|
| 515 |
-
// Torso
|
| 516 |
-
['chest', 'chest'], ['upperChest', 'chest'],
|
| 517 |
-
['spine', 'belly'],
|
| 518 |
-
['hips', 'buttocks'],
|
| 519 |
-
]
|
| 520 |
-
const _bonePos = new THREE.Vector3()
|
| 521 |
-
|
| 522 |
-
function detectTouchRegion(e: PointerEvent): TouchRegion | null {
|
| 523 |
-
if (!vrm?.humanoid) return null
|
| 524 |
-
|
| 525 |
-
touchMouseVec.set(
|
| 526 |
-
(e.clientX / window.innerWidth) * 2 - 1,
|
| 527 |
-
-(e.clientY / window.innerHeight) * 2 + 1,
|
| 528 |
-
)
|
| 529 |
-
touchRaycaster.setFromCamera(touchMouseVec, camera)
|
| 530 |
-
|
| 531 |
-
const intersects = touchRaycaster.intersectObject(vrm.scene, true)
|
| 532 |
-
if (intersects.length === 0) return null
|
| 533 |
-
|
| 534 |
-
const hitPoint = intersects[0].point
|
| 535 |
-
|
| 536 |
-
// Find closest bone to hit point
|
| 537 |
-
let closestRegion: TouchRegion = 'belly'
|
| 538 |
-
let closestDist = Infinity
|
| 539 |
-
|
| 540 |
-
for (const [boneName, region] of boneRegionMap) {
|
| 541 |
-
const bone = vrm.humanoid.getNormalizedBoneNode(boneName as any)
|
| 542 |
-
if (!bone) continue
|
| 543 |
-
bone.getWorldPosition(_bonePos)
|
| 544 |
-
const dist = hitPoint.distanceToSquared(_bonePos)
|
| 545 |
-
if (dist < closestDist) {
|
| 546 |
-
closestDist = dist
|
| 547 |
-
closestRegion = region
|
| 548 |
-
}
|
| 549 |
-
}
|
| 550 |
-
|
| 551 |
-
return closestRegion
|
| 552 |
-
}
|
| 553 |
-
|
| 554 |
-
// ── Left-click: distinguish click (touch) vs drag (move window) ────
|
| 555 |
-
let leftDownPos: { x: number; y: number; time: number; region: TouchRegion | null } | null = null
|
| 556 |
-
const CLICK_MOVE_THRESHOLD = 5 // px
|
| 557 |
-
const CLICK_TIME_THRESHOLD = 300 // ms
|
| 558 |
-
|
| 559 |
function onPointerDown(e: PointerEvent) {
|
| 560 |
-
if (e.button ===
|
| 561 |
-
const region = detectTouchRegion(e)
|
| 562 |
-
if (region) {
|
| 563 |
-
// Might be a touch — wait for pointerup to confirm it's not a drag
|
| 564 |
-
leftDownPos = { x: e.clientX, y: e.clientY, time: Date.now(), region }
|
| 565 |
-
return
|
| 566 |
-
}
|
| 567 |
-
// Not on model — no window drag available in web mode
|
| 568 |
-
return
|
| 569 |
-
} else if (e.button === 1) {
|
| 570 |
dragMode = 'dolly'
|
| 571 |
e.preventDefault()
|
| 572 |
} else if (e.button === 2) {
|
|
@@ -580,15 +451,6 @@ export const VRMScene = forwardRef<VRMSceneHandle, VRMSceneProps>(function VRMSc
|
|
| 580 |
}
|
| 581 |
|
| 582 |
function onPointerMove(e: PointerEvent) {
|
| 583 |
-
// If left button held on model and moved beyond threshold → it's a drag, not a touch
|
| 584 |
-
if (leftDownPos && e.buttons & 1) {
|
| 585 |
-
const dx = Math.abs(e.clientX - leftDownPos.x)
|
| 586 |
-
const dy = Math.abs(e.clientY - leftDownPos.y)
|
| 587 |
-
if (dx > CLICK_MOVE_THRESHOLD || dy > CLICK_MOVE_THRESHOLD) {
|
| 588 |
-
leftDownPos = null
|
| 589 |
-
return
|
| 590 |
-
}
|
| 591 |
-
}
|
| 592 |
if (!dragMode) return
|
| 593 |
const dx = e.clientX - prevX
|
| 594 |
const dy = e.clientY - prevY
|
|
@@ -608,42 +470,13 @@ export const VRMScene = forwardRef<VRMSceneHandle, VRMSceneProps>(function VRMSc
|
|
| 608 |
MIN_RADIUS,
|
| 609 |
MAX_RADIUS,
|
| 610 |
)
|
| 611 |
-
} else if (dragMode === 'pan') {
|
| 612 |
-
pivot.x -= dx * PAN_SPEED
|
| 613 |
-
pivot.y += dy * PAN_SPEED
|
| 614 |
}
|
| 615 |
updateCameraOrbit()
|
| 616 |
}
|
| 617 |
|
| 618 |
-
function onPointerUp(
|
| 619 |
-
// Confirm touch: short press with no movement on model
|
| 620 |
-
if (leftDownPos && e.button === 0) {
|
| 621 |
-
const elapsed = Date.now() - leftDownPos.time
|
| 622 |
-
const dx = Math.abs(e.clientX - leftDownPos.x)
|
| 623 |
-
const dy = Math.abs(e.clientY - leftDownPos.y)
|
| 624 |
-
if (elapsed < CLICK_TIME_THRESHOLD && dx <= CLICK_MOVE_THRESHOLD && dy <= CLICK_MOVE_THRESHOLD) {
|
| 625 |
-
const now = Date.now()
|
| 626 |
-
const region = leftDownPos.region!
|
| 627 |
-
if (now - lastTapTime < DOUBLE_TAP_WINDOW && lastTapRegion === region && now - lastTouchFireTime > TOUCH_COOLDOWN) {
|
| 628 |
-
// Double-tap confirmed
|
| 629 |
-
lastTouchFireTime = now
|
| 630 |
-
spawnRipple(e.clientX, e.clientY, true)
|
| 631 |
-
onTouchRef.current?.(region)
|
| 632 |
-
lastTapTime = 0
|
| 633 |
-
lastTapRegion = null
|
| 634 |
-
} else {
|
| 635 |
-
// First tap — wait for second
|
| 636 |
-
spawnRipple(e.clientX, e.clientY, false)
|
| 637 |
-
lastTapTime = now
|
| 638 |
-
lastTapRegion = region
|
| 639 |
-
}
|
| 640 |
-
}
|
| 641 |
-
leftDownPos = null
|
| 642 |
-
return
|
| 643 |
-
}
|
| 644 |
if (dragMode) {
|
| 645 |
dragMode = null
|
| 646 |
-
canvas!.releasePointerCapture(e.pointerId)
|
| 647 |
}
|
| 648 |
}
|
| 649 |
|
|
@@ -664,20 +497,6 @@ export const VRMScene = forwardRef<VRMSceneHandle, VRMSceneProps>(function VRMSc
|
|
| 664 |
}
|
| 665 |
window.addEventListener('resize', onResize)
|
| 666 |
|
| 667 |
-
// ── Hit-test for window pass-through ──────────────────────────────────────
|
| 668 |
-
// Offscreen render target: render scene, read 1 pixel alpha at cursor.
|
| 669 |
-
// Updated in the render loop — no extra render pass, just piggybacks.
|
| 670 |
-
const hitTarget = new THREE.WebGLRenderTarget(1, 1, { depthBuffer: false })
|
| 671 |
-
let pendingHitTest: { x: number; y: number; resolve: (hit: boolean) => void } | null = null
|
| 672 |
-
const hitPixel = new Uint8Array(4)
|
| 673 |
-
|
| 674 |
-
// Async hit-test: queues a request, resolved after next frame render
|
| 675 |
-
;(window as any).__clawHitTest = (clientX: number, clientY: number): Promise<boolean> => {
|
| 676 |
-
return new Promise((resolve) => {
|
| 677 |
-
pendingHitTest = { x: clientX, y: clientY, resolve }
|
| 678 |
-
})
|
| 679 |
-
}
|
| 680 |
-
|
| 681 |
// ── Animation loop ────────────────────────────────────────────────────────
|
| 682 |
const clock = new THREE.Clock()
|
| 683 |
let animFrameId: number
|
|
@@ -687,16 +506,12 @@ export const VRMScene = forwardRef<VRMSceneHandle, VRMSceneProps>(function VRMSc
|
|
| 687 |
const delta = clock.getDelta()
|
| 688 |
|
| 689 |
if (vrm) {
|
| 690 |
-
// 1. Animation mixer
|
| 691 |
motion?.update(delta)
|
| 692 |
|
| 693 |
-
|
| 694 |
-
if (handPose && !motion?.isDancing) applyRelaxedHandPose(handPose, clock.elapsedTime)
|
| 695 |
|
| 696 |
-
// 2. Humanoid update
|
| 697 |
vrm.humanoid?.update()
|
| 698 |
|
| 699 |
-
// 3. Camera tracking mode: look at camera position
|
| 700 |
if (trackingModeRef.current === 'camera') {
|
| 701 |
lookAtTarget.x = camera.position.x
|
| 702 |
lookAtTarget.y = camera.position.y
|
|
@@ -704,60 +519,84 @@ export const VRMScene = forwardRef<VRMSceneHandle, VRMSceneProps>(function VRMSc
|
|
| 704 |
saccades.instantUpdate(vrm, lookAtTarget)
|
| 705 |
}
|
| 706 |
|
| 707 |
-
// 4. LookAt update
|
| 708 |
vrm.lookAt?.update(delta)
|
| 709 |
-
|
| 710 |
-
// 5. Eye saccades (airi style)
|
| 711 |
saccades.update(vrm, lookAtTarget, delta)
|
| 712 |
-
|
| 713 |
-
// 5. Blinking
|
| 714 |
updateBlink(vrm, delta, blinkState)
|
| 715 |
-
|
| 716 |
-
// 6. Emote transitions
|
| 717 |
emote?.update(delta)
|
| 718 |
-
|
| 719 |
-
// 7. Lip sync
|
| 720 |
lipSyncRef.current.update(vrm, delta)
|
| 721 |
|
| 722 |
-
//
|
| 723 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 724 |
|
| 725 |
-
|
| 726 |
vrm.springBoneManager?.update(delta)
|
| 727 |
-
}
|
| 728 |
-
|
| 729 |
-
renderer.render(scene, camera)
|
| 730 |
|
| 731 |
-
|
| 732 |
-
|
| 733 |
-
const
|
| 734 |
-
|
| 735 |
-
|
| 736 |
-
if (
|
| 737 |
-
|
| 738 |
-
}
|
| 739 |
|
| 740 |
-
|
| 741 |
-
|
| 742 |
-
const
|
|
|
|
|
|
|
|
|
|
| 743 |
|
| 744 |
-
|
| 745 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 746 |
}
|
| 747 |
|
| 748 |
-
//
|
| 749 |
-
|
| 750 |
-
|
| 751 |
-
|
| 752 |
-
|
| 753 |
-
|
| 754 |
-
|
| 755 |
-
|
| 756 |
-
|
| 757 |
-
|
| 758 |
-
|
| 759 |
-
}
|
| 760 |
}
|
|
|
|
|
|
|
| 761 |
}
|
| 762 |
|
| 763 |
animate()
|
|
@@ -777,7 +616,6 @@ export const VRMScene = forwardRef<VRMSceneHandle, VRMSceneProps>(function VRMSc
|
|
| 777 |
emoteRef.current = null
|
| 778 |
motion?.dispose()
|
| 779 |
motionRef.current = null
|
| 780 |
-
hitTarget.dispose()
|
| 781 |
delete (window as any).__clawHitTest
|
| 782 |
renderer.dispose()
|
| 783 |
}
|
|
@@ -795,18 +633,11 @@ export const VRMScene = forwardRef<VRMSceneHandle, VRMSceneProps>(function VRMSc
|
|
| 795 |
cursor: 'grab',
|
| 796 |
}}
|
| 797 |
/>
|
| 798 |
-
{ripples.map(r => (
|
| 799 |
-
<span
|
| 800 |
-
key={r.id}
|
| 801 |
-
className={r.confirmed ? 'touch-ripple confirmed' : 'touch-ripple'}
|
| 802 |
-
style={{ left: r.x, top: r.y }}
|
| 803 |
-
/>
|
| 804 |
-
))}
|
| 805 |
</div>
|
| 806 |
)
|
| 807 |
})
|
| 808 |
|
| 809 |
-
// ── Eye saccade interval
|
| 810 |
const EYE_SACCADE_INT_STEP = 400
|
| 811 |
const EYE_SACCADE_INT_P: number[][] = [
|
| 812 |
[0.075, 800], [0.110, 0], [0.125, 0], [0.140, 0], [0.125, 0],
|
|
@@ -827,13 +658,11 @@ function randomSaccadeInterval(): number {
|
|
| 827 |
return EYE_SACCADE_INT_P[EYE_SACCADE_INT_P.length - 1][1] + Math.random() * EYE_SACCADE_INT_STEP
|
| 828 |
}
|
| 829 |
|
| 830 |
-
// ── Eye saccade controller (from airi) ───────────────────────────────────────
|
| 831 |
class EyeSaccadeController {
|
| 832 |
private nextSaccadeAfter = -1
|
| 833 |
private timeSinceLastSaccade = 0
|
| 834 |
private fixationTarget = new THREE.Vector3()
|
| 835 |
|
| 836 |
-
/** Called when lookAt target changes (e.g. mouse moved) */
|
| 837 |
instantUpdate(vrm: VRM, target: { x: number; y: number; z: number }) {
|
| 838 |
this.fixationTarget.set(target.x, target.y, target.z)
|
| 839 |
if (!vrm.lookAt) return
|
|
@@ -844,12 +673,10 @@ class EyeSaccadeController {
|
|
| 844 |
vrm.lookAt.update(0.016)
|
| 845 |
}
|
| 846 |
|
| 847 |
-
/** Called every frame */
|
| 848 |
update(vrm: VRM, lookAtTarget: { x: number; y: number; z: number }, delta: number) {
|
| 849 |
if (!vrm.expressionManager || !vrm.lookAt) return
|
| 850 |
|
| 851 |
if (this.timeSinceLastSaccade >= this.nextSaccadeAfter) {
|
| 852 |
-
// Add random offset to the current lookAt target
|
| 853 |
this.fixationTarget.set(
|
| 854 |
lookAtTarget.x + THREE.MathUtils.randFloat(-0.25, 0.25),
|
| 855 |
lookAtTarget.y + THREE.MathUtils.randFloat(-0.25, 0.25),
|
|
|
|
| 1 |
+
import { useEffect, useRef, useImperativeHandle, forwardRef } from 'react'
|
| 2 |
import * as THREE from 'three'
|
| 3 |
import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js'
|
| 4 |
import { VRMLoaderPlugin, VRMUtils } from '@pixiv/three-vrm'
|
|
|
|
| 8 |
import { LipSync } from '../lip-sync'
|
| 9 |
import { MotionController } from '../motion-controller'
|
| 10 |
|
|
|
|
|
|
|
| 11 |
interface VRMSceneProps {
|
| 12 |
modelPath: string
|
| 13 |
idleAnimationPath?: string
|
|
|
|
|
|
|
| 14 |
}
|
| 15 |
|
| 16 |
export type TrackingMode = 'mouse' | 'camera'
|
|
|
|
| 21 |
resetCamera: () => void
|
| 22 |
setTrackingMode: (mode: TrackingMode) => void
|
| 23 |
playAction: (name: string, hold?: boolean) => void
|
|
|
|
| 24 |
panCamera: (dx: number, dy: number) => void
|
| 25 |
rotateCamera: (dx: number, dy: number) => void
|
|
|
|
|
|
|
|
|
|
|
|
|
| 26 |
/** Unified reset: camera + resetToIdle + expressions to zero */
|
| 27 |
reset: () => void
|
| 28 |
}
|
|
|
|
| 33 |
blinkProgress: number
|
| 34 |
timeSinceLastBlink: number
|
| 35 |
nextBlinkTime: number
|
| 36 |
+
blinkDuration: number
|
| 37 |
+
doubleBlink: boolean
|
| 38 |
+
doubleBlinkCount: number
|
| 39 |
}
|
| 40 |
|
| 41 |
function createBlinkState(): BlinkState {
|
|
|
|
| 43 |
isBlinking: false,
|
| 44 |
blinkProgress: 0,
|
| 45 |
timeSinceLastBlink: 0,
|
| 46 |
+
nextBlinkTime: Math.random() * 5 + 2,
|
| 47 |
+
blinkDuration: 0.1 + Math.random() * 0.08,
|
| 48 |
+
doubleBlink: false,
|
| 49 |
+
doubleBlinkCount: 0,
|
| 50 |
}
|
| 51 |
}
|
| 52 |
|
|
|
|
| 58 |
if (!state.isBlinking && state.timeSinceLastBlink >= state.nextBlinkTime) {
|
| 59 |
state.isBlinking = true
|
| 60 |
state.blinkProgress = 0
|
| 61 |
+
state.blinkDuration = 0.08 + Math.random() * 0.12
|
| 62 |
+
// ~8% chance of double blink (more natural)
|
| 63 |
+
state.doubleBlink = Math.random() < 0.08
|
| 64 |
+
state.doubleBlinkCount = 0
|
| 65 |
}
|
| 66 |
|
| 67 |
if (state.isBlinking) {
|
| 68 |
+
state.blinkProgress += delta / state.blinkDuration
|
|
|
|
| 69 |
const blinkValue = Math.sin(Math.PI * state.blinkProgress)
|
| 70 |
vrm.expressionManager.setValue('blink', blinkValue)
|
| 71 |
|
| 72 |
if (state.blinkProgress >= 1) {
|
| 73 |
+
state.blinkProgress = 0
|
| 74 |
+
state.doubleBlinkCount++
|
| 75 |
+
|
| 76 |
+
if (state.doubleBlink && state.doubleBlinkCount < 2) {
|
| 77 |
+
// Quick reopen then re-blink
|
| 78 |
+
vrm.expressionManager.setValue('blink', 0)
|
| 79 |
+
state.blinkDuration = 0.06 + Math.random() * 0.06
|
| 80 |
+
} else {
|
| 81 |
+
state.isBlinking = false
|
| 82 |
+
state.timeSinceLastBlink = 0
|
| 83 |
+
vrm.expressionManager.setValue('blink', 0)
|
| 84 |
+
state.doubleBlink = false
|
| 85 |
+
state.nextBlinkTime = Math.random() * 6 + 1.5
|
| 86 |
+
}
|
| 87 |
}
|
| 88 |
}
|
| 89 |
}
|
| 90 |
|
| 91 |
// ── Relaxed hand pose ─────────────────────────────────────────────────────────
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 92 |
|
| 93 |
interface HandPoseCache {
|
| 94 |
bones: { bone: THREE.Object3D; z: number; y: number }[]
|
|
|
|
| 103 |
const segments = ['Proximal', 'Intermediate', 'Distal'] as const
|
| 104 |
const sides = ['left', 'right'] as const
|
| 105 |
|
|
|
|
| 106 |
const curlMap: Record<string, [number, number, number]> = {
|
| 107 |
Thumb: [0.25, 0.15, 0.10],
|
| 108 |
Index: [0.20, 0.30, 0.20],
|
|
|
|
| 111 |
Little: [0.35, 0.45, 0.30],
|
| 112 |
}
|
| 113 |
|
|
|
|
| 114 |
const spreadMap: Record<string, number> = {
|
| 115 |
Thumb: 0.15,
|
| 116 |
Index: 0.04,
|
|
|
|
| 132 |
if (!bone) continue
|
| 133 |
|
| 134 |
const z = sign * curls[s]
|
|
|
|
| 135 |
const y = s === 0 ? sign * spread : 0
|
| 136 |
|
| 137 |
bones.push({ bone, z, y })
|
|
|
|
| 143 |
}
|
| 144 |
|
| 145 |
function applyRelaxedHandPose(cache: HandPoseCache, time: number) {
|
| 146 |
+
// Slow hand tension cycle (~30s period): hands subtly change posture over time
|
| 147 |
+
const tensionCycle = 0.5 + 0.5 * Math.sin(time * 0.035)
|
| 148 |
+
const zScale = 0.85 + tensionCycle * 0.3 // 0.85-1.15
|
| 149 |
+
|
| 150 |
for (const { bone, z, y } of cache.bones) {
|
|
|
|
|
|
|
| 151 |
const freq = 0.3 + Math.abs(z) * 2
|
| 152 |
const micro = Math.sin(time * freq + z * 50) * 0.02
|
| 153 |
+
bone.rotation.z = z * zScale + micro
|
| 154 |
if (y !== 0) bone.rotation.y = y
|
| 155 |
}
|
| 156 |
}
|
| 157 |
|
|
|
|
| 158 |
export const VRMScene = forwardRef<VRMSceneHandle, VRMSceneProps>(function VRMScene({
|
| 159 |
modelPath,
|
| 160 |
idleAnimationPath = '/friend/idle_loop.vrma',
|
|
|
|
|
|
|
| 161 |
}, ref) {
|
| 162 |
const canvasRef = useRef<HTMLCanvasElement>(null)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 163 |
|
| 164 |
const emoteRef = useRef<EmoteController | null>(null)
|
| 165 |
const resetCameraRef = useRef<(() => void) | null>(null)
|
|
|
|
| 168 |
const panCameraRef = useRef<((dx: number, dy: number) => void) | null>(null)
|
| 169 |
const rotateCameraRef = useRef<((dx: number, dy: number) => void) | null>(null)
|
| 170 |
const lipSyncRef = useRef<LipSync>(LipSync.getInstance())
|
|
|
|
|
|
|
|
|
|
|
|
|
| 171 |
|
| 172 |
useImperativeHandle(ref, () => ({
|
| 173 |
setEmotion(emotion: string, intensity?: number) {
|
|
|
|
| 185 |
playAction(name: string, hold?: boolean) {
|
| 186 |
motionRef.current?.playAction(name, hold)
|
| 187 |
},
|
|
|
|
|
|
|
|
|
|
| 188 |
panCamera(dx: number, dy: number) {
|
| 189 |
panCameraRef.current?.(dx, dy)
|
| 190 |
},
|
| 191 |
rotateCamera(dx: number, dy: number) {
|
| 192 |
rotateCameraRef.current?.(dx, dy)
|
| 193 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 194 |
reset() {
|
| 195 |
resetCameraRef.current?.()
|
| 196 |
motionRef.current?.resetToIdle()
|
|
|
|
| 224 |
0.1,
|
| 225 |
100,
|
| 226 |
)
|
|
|
|
|
|
|
| 227 |
const pivot = new THREE.Vector3(0, 0, 0)
|
| 228 |
let orbitRadius = 2.0
|
| 229 |
+
let orbitTheta = 0
|
| 230 |
+
let orbitPhi = Math.PI / 2
|
| 231 |
|
| 232 |
function updateCameraOrbit() {
|
| 233 |
camera.position.set(
|
|
|
|
| 261 |
const saccades = new EyeSaccadeController()
|
| 262 |
const lookAtTarget = { x: 0, y: 0, z: -100 }
|
| 263 |
|
| 264 |
+
// ── Aliveness system state: breathing, sway, speech micro-movements ─────
|
| 265 |
+
let alivenessBones = { chestBone: null as THREE.Object3D | null, spineBone: null as THREE.Object3D | null, neckBone: null as THREE.Object3D | null }
|
| 266 |
+
let breathPhase = Math.random() * Math.PI * 2
|
| 267 |
+
let swayPhase = Math.random() * Math.PI * 2
|
| 268 |
+
let speechBlend = 0
|
| 269 |
+
let userListenBlend = 0
|
| 270 |
+
// Micro-expression state
|
| 271 |
+
let microTimer = 5 + Math.random() * 10
|
| 272 |
+
let microActive = false
|
| 273 |
+
let microPhase = 0
|
| 274 |
+
let microShape = ''
|
| 275 |
+
|
| 276 |
// ── Load VRM model, then load idle animation ─────────────────────────────
|
| 277 |
loader.load(
|
| 278 |
modelPath,
|
|
|
|
| 289 |
obj.frustumCulled = false
|
| 290 |
})
|
| 291 |
|
|
|
|
| 292 |
if (loadedVrm.lookAt) {
|
| 293 |
const lookAtQuatProxy = new VRMLookAtQuaternionProxy(loadedVrm.lookAt)
|
| 294 |
lookAtQuatProxy.name = 'lookAtQuaternionProxy'
|
| 295 |
loadedVrm.scene.add(lookAtQuatProxy)
|
| 296 |
}
|
| 297 |
|
|
|
|
| 298 |
VRMUtils.rotateVRM0(loadedVrm)
|
| 299 |
|
| 300 |
scene.add(loadedVrm.scene)
|
| 301 |
vrm = loadedVrm
|
| 302 |
|
| 303 |
+
// ── Compute camera from model bounds ───────────────────
|
| 304 |
const box = new THREE.Box3().setFromObject(loadedVrm.scene)
|
| 305 |
const modelSize = new THREE.Vector3()
|
| 306 |
const modelCenter = new THREE.Vector3()
|
| 307 |
box.getSize(modelSize)
|
| 308 |
box.getCenter(modelCenter)
|
| 309 |
+
modelCenter.y += modelSize.y / 3.2
|
| 310 |
|
| 311 |
const radians = (FOV / 2 * Math.PI) / 180
|
| 312 |
const offsetX = modelSize.x / 16
|
|
|
|
| 319 |
orbitPhi = Math.PI / 2 - Math.atan2(offsetY, offsetZ)
|
| 320 |
updateCameraOrbit()
|
| 321 |
|
|
|
|
| 322 |
const initPivot = pivot.clone()
|
| 323 |
const initRadius = orbitRadius
|
| 324 |
const initTheta = orbitTheta
|
|
|
|
| 335 |
const right = new THREE.Vector3()
|
| 336 |
const up = new THREE.Vector3()
|
| 337 |
camera.getWorldDirection(new THREE.Vector3())
|
| 338 |
+
right.setFromMatrixColumn(camera.matrixWorld, 0)
|
| 339 |
+
up.setFromMatrixColumn(camera.matrixWorld, 1)
|
| 340 |
pivot.addScaledVector(right, -dx * 0.003)
|
| 341 |
pivot.addScaledVector(up, dy * 0.003)
|
| 342 |
updateCameraOrbit()
|
|
|
|
| 352 |
updateCameraOrbit()
|
| 353 |
}
|
| 354 |
|
|
|
|
| 355 |
handPose = buildHandPoseCache(loadedVrm)
|
| 356 |
|
| 357 |
+
// ── Initialize aliveness bone references ──
|
| 358 |
+
const h = loadedVrm.humanoid
|
| 359 |
+
alivenessBones = {
|
| 360 |
+
chestBone: h?.getNormalizedBoneNode('chest') ?? null,
|
| 361 |
+
spineBone: h?.getNormalizedBoneNode('spine') ?? null,
|
| 362 |
+
neckBone: h?.getNormalizedBoneNode('neck') ?? null,
|
| 363 |
+
}
|
| 364 |
+
|
| 365 |
emote = new EmoteController(loadedVrm)
|
| 366 |
emoteRef.current = emote
|
| 367 |
|
|
|
|
| 368 |
motion = new MotionController(loadedVrm)
|
| 369 |
motionRef.current = motion
|
| 370 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 371 |
motion.loadIdle(idleAnimationPath).catch((err) =>
|
| 372 |
console.warn('Failed to load idle animation:', err),
|
| 373 |
)
|
| 374 |
|
|
|
|
| 375 |
loadedVrm.springBoneManager?.reset()
|
|
|
|
|
|
|
|
|
|
|
|
|
| 376 |
},
|
| 377 |
() => {},
|
| 378 |
(err) => {
|
|
|
|
| 382 |
|
| 383 |
// ── Mouse tracking ────────────────────────────────────────────────────────
|
| 384 |
const mouse = new THREE.Vector2(0, 0)
|
|
|
|
| 385 |
const _raycaster = new THREE.Raycaster()
|
| 386 |
const _mouseVec = new THREE.Vector2()
|
| 387 |
|
|
|
|
| 391 |
|
| 392 |
if (trackingModeRef.current !== 'mouse') return
|
| 393 |
|
|
|
|
| 394 |
_mouseVec.set(mouse.x, mouse.y)
|
| 395 |
_raycaster.setFromCamera(_mouseVec, camera)
|
| 396 |
const camDir = new THREE.Vector3()
|
|
|
|
| 410 |
}
|
| 411 |
}
|
| 412 |
}
|
|
|
|
| 413 |
window.addEventListener('mousemove', onMouseMove)
|
| 414 |
document.addEventListener('mousemove', onMouseMove)
|
| 415 |
|
|
|
|
| 429 |
}
|
| 430 |
canvas.addEventListener('wheel', onWheel, { passive: false })
|
| 431 |
|
| 432 |
+
// ── Camera drag controls ──────────────────────────────────────────────
|
| 433 |
+
let dragMode: 'rotate' | 'dolly' | null = null
|
|
|
|
|
|
|
|
|
|
| 434 |
let prevX = 0
|
| 435 |
let prevY = 0
|
| 436 |
const ROTATE_SPEED = 0.005
|
|
|
|
| 437 |
const DOLLY_SPEED = 0.01
|
| 438 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 439 |
function onPointerDown(e: PointerEvent) {
|
| 440 |
+
if (e.button === 1) {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 441 |
dragMode = 'dolly'
|
| 442 |
e.preventDefault()
|
| 443 |
} else if (e.button === 2) {
|
|
|
|
| 451 |
}
|
| 452 |
|
| 453 |
function onPointerMove(e: PointerEvent) {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 454 |
if (!dragMode) return
|
| 455 |
const dx = e.clientX - prevX
|
| 456 |
const dy = e.clientY - prevY
|
|
|
|
| 470 |
MIN_RADIUS,
|
| 471 |
MAX_RADIUS,
|
| 472 |
)
|
|
|
|
|
|
|
|
|
|
| 473 |
}
|
| 474 |
updateCameraOrbit()
|
| 475 |
}
|
| 476 |
|
| 477 |
+
function onPointerUp() {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 478 |
if (dragMode) {
|
| 479 |
dragMode = null
|
|
|
|
| 480 |
}
|
| 481 |
}
|
| 482 |
|
|
|
|
| 497 |
}
|
| 498 |
window.addEventListener('resize', onResize)
|
| 499 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 500 |
// ── Animation loop ────────────────────────────────────────────────────────
|
| 501 |
const clock = new THREE.Clock()
|
| 502 |
let animFrameId: number
|
|
|
|
| 506 |
const delta = clock.getDelta()
|
| 507 |
|
| 508 |
if (vrm) {
|
|
|
|
| 509 |
motion?.update(delta)
|
| 510 |
|
| 511 |
+
if (handPose) applyRelaxedHandPose(handPose, clock.elapsedTime)
|
|
|
|
| 512 |
|
|
|
|
| 513 |
vrm.humanoid?.update()
|
| 514 |
|
|
|
|
| 515 |
if (trackingModeRef.current === 'camera') {
|
| 516 |
lookAtTarget.x = camera.position.x
|
| 517 |
lookAtTarget.y = camera.position.y
|
|
|
|
| 519 |
saccades.instantUpdate(vrm, lookAtTarget)
|
| 520 |
}
|
| 521 |
|
|
|
|
| 522 |
vrm.lookAt?.update(delta)
|
|
|
|
|
|
|
| 523 |
saccades.update(vrm, lookAtTarget, delta)
|
|
|
|
|
|
|
| 524 |
updateBlink(vrm, delta, blinkState)
|
|
|
|
|
|
|
| 525 |
emote?.update(delta)
|
|
|
|
|
|
|
| 526 |
lipSyncRef.current.update(vrm, delta)
|
| 527 |
|
| 528 |
+
// ── Aliveness: micro-expressions (asymmetric blink, subtle morphs) ──
|
| 529 |
+
microTimer -= delta
|
| 530 |
+
if (microTimer <= 0 && !microActive) {
|
| 531 |
+
microShape = Math.random() > 0.5 ? 'blinkLeft' : 'blinkRight'
|
| 532 |
+
microActive = true
|
| 533 |
+
microPhase = 0
|
| 534 |
+
}
|
| 535 |
+
if (microActive && vrm.expressionManager) {
|
| 536 |
+
microPhase += delta * 4
|
| 537 |
+
const val = Math.sin(Math.PI * Math.min(microPhase, 1))
|
| 538 |
+
vrm.expressionManager.setValue(microShape, val * 0.3)
|
| 539 |
+
if (microPhase >= 2) {
|
| 540 |
+
microActive = false
|
| 541 |
+
vrm.expressionManager.setValue(microShape, 0)
|
| 542 |
+
microTimer = 8 + Math.random() * 16
|
| 543 |
+
}
|
| 544 |
+
}
|
| 545 |
|
| 546 |
+
vrm.expressionManager?.update()
|
| 547 |
vrm.springBoneManager?.update(delta)
|
|
|
|
|
|
|
|
|
|
| 548 |
|
| 549 |
+
// ── Aliveness: breathing (chest rise/fall, post-mixer) ──
|
| 550 |
+
// Breathing rate varies naturally, amplitude tuned for VRM scale
|
| 551 |
+
const breathRate = 1.8 + Math.sin(breathPhase * 0.05) * 0.4 + Math.sin(clock.elapsedTime * 0.1) * 0.3
|
| 552 |
+
breathPhase += delta * breathRate
|
| 553 |
+
const breathVal = Math.sin(breathPhase) * 0.006
|
| 554 |
+
if (alivenessBones.chestBone) {
|
| 555 |
+
alivenessBones.chestBone.position.y += breathVal
|
| 556 |
+
}
|
| 557 |
|
| 558 |
+
// ── Aliveness: postural sway ──
|
| 559 |
+
swayPhase += delta * 0.35
|
| 560 |
+
const swayZ = Math.sin(swayPhase) * 0.005
|
| 561 |
+
if (alivenessBones.spineBone) {
|
| 562 |
+
alivenessBones.spineBone.rotation.z += swayZ
|
| 563 |
+
}
|
| 564 |
|
| 565 |
+
// ── Aliveness: speech-driven micro-movements ──
|
| 566 |
+
const isSpeaking = lipSyncRef.current.isActive()
|
| 567 |
+
const targetBlend = isSpeaking ? 1 : 0
|
| 568 |
+
speechBlend += (targetBlend - speechBlend) * Math.min(1, delta * 3)
|
| 569 |
+
if (speechBlend > 0.01) {
|
| 570 |
+
const t = clock.elapsedTime
|
| 571 |
+
const headX = Math.sin(t * 3.7 + 1.2) * 0.02 * speechBlend
|
| 572 |
+
const headZ = Math.sin(t * 2.3 + 0.7) * 0.015 * speechBlend
|
| 573 |
+
const headY = Math.sin(t * 1.5 + 3.8) * 0.008 * speechBlend // slight rotation (looking around while talking)
|
| 574 |
+
const spineSway = Math.sin(t * 1.8 + 0.3) * 0.006 * speechBlend
|
| 575 |
+
if (alivenessBones.neckBone) {
|
| 576 |
+
alivenessBones.neckBone.rotation.x += headX
|
| 577 |
+
alivenessBones.neckBone.rotation.z += headZ
|
| 578 |
+
alivenessBones.neckBone.rotation.y += headY
|
| 579 |
+
}
|
| 580 |
+
if (alivenessBones.spineBone) {
|
| 581 |
+
alivenessBones.spineBone.rotation.x += spineSway
|
| 582 |
+
}
|
| 583 |
}
|
| 584 |
|
| 585 |
+
// ── Aliveness: listening response when user is speaking ──
|
| 586 |
+
const isUserSpeaking = !!(window as any).__userRecording
|
| 587 |
+
userListenBlend += ((isUserSpeaking ? 1 : 0) - userListenBlend) * Math.min(1, delta * 2)
|
| 588 |
+
if (userListenBlend > 0.01) {
|
| 589 |
+
const t = clock.elapsedTime
|
| 590 |
+
const listenTilt = Math.sin(t * 0.9 + 0.5) * 0.012 * userListenBlend // head tilt
|
| 591 |
+
const listenNod = Math.sin(t * 2.3 + 3.1) * 0.008 * userListenBlend // subtle nodding
|
| 592 |
+
if (alivenessBones.neckBone) {
|
| 593 |
+
alivenessBones.neckBone.rotation.z += listenTilt
|
| 594 |
+
alivenessBones.neckBone.rotation.x += listenNod
|
| 595 |
+
}
|
| 596 |
+
}
|
| 597 |
}
|
| 598 |
+
|
| 599 |
+
renderer.render(scene, camera)
|
| 600 |
}
|
| 601 |
|
| 602 |
animate()
|
|
|
|
| 616 |
emoteRef.current = null
|
| 617 |
motion?.dispose()
|
| 618 |
motionRef.current = null
|
|
|
|
| 619 |
delete (window as any).__clawHitTest
|
| 620 |
renderer.dispose()
|
| 621 |
}
|
|
|
|
| 633 |
cursor: 'grab',
|
| 634 |
}}
|
| 635 |
/>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 636 |
</div>
|
| 637 |
)
|
| 638 |
})
|
| 639 |
|
| 640 |
+
// ── Eye saccade interval ─────────────────────────────────────────
|
| 641 |
const EYE_SACCADE_INT_STEP = 400
|
| 642 |
const EYE_SACCADE_INT_P: number[][] = [
|
| 643 |
[0.075, 800], [0.110, 0], [0.125, 0], [0.140, 0], [0.125, 0],
|
|
|
|
| 658 |
return EYE_SACCADE_INT_P[EYE_SACCADE_INT_P.length - 1][1] + Math.random() * EYE_SACCADE_INT_STEP
|
| 659 |
}
|
| 660 |
|
|
|
|
| 661 |
class EyeSaccadeController {
|
| 662 |
private nextSaccadeAfter = -1
|
| 663 |
private timeSinceLastSaccade = 0
|
| 664 |
private fixationTarget = new THREE.Vector3()
|
| 665 |
|
|
|
|
| 666 |
instantUpdate(vrm: VRM, target: { x: number; y: number; z: number }) {
|
| 667 |
this.fixationTarget.set(target.x, target.y, target.z)
|
| 668 |
if (!vrm.lookAt) return
|
|
|
|
| 673 |
vrm.lookAt.update(0.016)
|
| 674 |
}
|
| 675 |
|
|
|
|
| 676 |
update(vrm: VRM, lookAtTarget: { x: number; y: number; z: number }, delta: number) {
|
| 677 |
if (!vrm.expressionManager || !vrm.lookAt) return
|
| 678 |
|
| 679 |
if (this.timeSinceLastSaccade >= this.nextSaccadeAfter) {
|
|
|
|
| 680 |
this.fixationTarget.set(
|
| 681 |
lookAtTarget.x + THREE.MathUtils.randFloat(-0.25, 0.25),
|
| 682 |
lookAtTarget.y + THREE.MathUtils.randFloat(-0.25, 0.25),
|
src/components/friend/frontend/index.html
CHANGED
|
@@ -52,45 +52,6 @@
|
|
| 52 |
100% { opacity: 1; transform: scale(1) translateY(0); }
|
| 53 |
}
|
| 54 |
|
| 55 |
-
@keyframes mood-bubble-float {
|
| 56 |
-
0% { opacity: 1; transform: translateX(-50%) translateY(0) scale(0.8); }
|
| 57 |
-
15% { opacity: 1; transform: translateX(-50%) translateY(-8px) scale(1.1); }
|
| 58 |
-
30% { opacity: 1; transform: translateX(-50%) translateY(-16px) scale(1); }
|
| 59 |
-
70% { opacity: 0.9; }
|
| 60 |
-
100% { opacity: 0; transform: translateX(-50%) translateY(-50px) scale(0.9); }
|
| 61 |
-
}
|
| 62 |
-
|
| 63 |
-
/* Touch ripple */
|
| 64 |
-
.touch-ripple {
|
| 65 |
-
position: fixed;
|
| 66 |
-
pointer-events: none;
|
| 67 |
-
width: 40px;
|
| 68 |
-
height: 40px;
|
| 69 |
-
margin-left: -20px;
|
| 70 |
-
margin-top: -20px;
|
| 71 |
-
border-radius: 50%;
|
| 72 |
-
border: 2px solid rgba(255, 255, 255, 0.5);
|
| 73 |
-
animation: ripple-expand 0.5s ease-out forwards;
|
| 74 |
-
}
|
| 75 |
-
.touch-ripple.confirmed {
|
| 76 |
-
width: 60px;
|
| 77 |
-
height: 60px;
|
| 78 |
-
margin-left: -30px;
|
| 79 |
-
margin-top: -30px;
|
| 80 |
-
border: 2.5px solid rgba(255, 180, 220, 0.8);
|
| 81 |
-
box-shadow: 0 0 12px rgba(255, 180, 220, 0.4);
|
| 82 |
-
animation: ripple-expand-confirmed 0.7s ease-out forwards;
|
| 83 |
-
}
|
| 84 |
-
|
| 85 |
-
@keyframes ripple-expand {
|
| 86 |
-
0% { transform: scale(0); opacity: 0.7; }
|
| 87 |
-
100% { transform: scale(1.8); opacity: 0; }
|
| 88 |
-
}
|
| 89 |
-
@keyframes ripple-expand-confirmed {
|
| 90 |
-
0% { transform: scale(0); opacity: 0.9; }
|
| 91 |
-
50% { transform: scale(1.2); opacity: 0.6; }
|
| 92 |
-
100% { transform: scale(2.2); opacity: 0; }
|
| 93 |
-
}
|
| 94 |
</style>
|
| 95 |
</head>
|
| 96 |
|
|
|
|
| 52 |
100% { opacity: 1; transform: scale(1) translateY(0); }
|
| 53 |
}
|
| 54 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 55 |
</style>
|
| 56 |
</head>
|
| 57 |
|
src/components/friend/frontend/lip-sync.ts
CHANGED
|
@@ -167,6 +167,11 @@ export class LipSync {
|
|
| 167 |
}
|
| 168 |
}
|
| 169 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 170 |
/** Stop current audio playback immediately. */
|
| 171 |
stopAudio() {
|
| 172 |
if (this.currentSource) {
|
|
|
|
| 167 |
}
|
| 168 |
}
|
| 169 |
|
| 170 |
+
/** Check if currently playing audio (within last 300ms). */
|
| 171 |
+
isActive(): boolean {
|
| 172 |
+
return this.currentSource !== null && performance.now() - this.lastActiveAt < 300
|
| 173 |
+
}
|
| 174 |
+
|
| 175 |
/** Stop current audio playback immediately. */
|
| 176 |
stopAudio() {
|
| 177 |
if (this.currentSource) {
|
src/components/friend/frontend/mmd-parser.d.ts
DELETED
|
@@ -1,39 +0,0 @@
|
|
| 1 |
-
declare module 'mmd-parser' {
|
| 2 |
-
export const CharsetEncoder: any
|
| 3 |
-
|
| 4 |
-
export class Parser {
|
| 5 |
-
parseVmd(buffer: ArrayBufferLike, leftToRight?: boolean): VmdFile
|
| 6 |
-
}
|
| 7 |
-
|
| 8 |
-
export interface VmdFile {
|
| 9 |
-
metadata: {
|
| 10 |
-
magic: string
|
| 11 |
-
name: string
|
| 12 |
-
motionCount: number
|
| 13 |
-
morphCount: number
|
| 14 |
-
cameraCount: number
|
| 15 |
-
coordinateSystem: string
|
| 16 |
-
}
|
| 17 |
-
motions: {
|
| 18 |
-
boneName: string
|
| 19 |
-
frameNum: number
|
| 20 |
-
position: number[]
|
| 21 |
-
rotation: number[]
|
| 22 |
-
interpolation: number[]
|
| 23 |
-
}[]
|
| 24 |
-
morphs: {
|
| 25 |
-
morphName: string
|
| 26 |
-
frameNum: number
|
| 27 |
-
weight: number
|
| 28 |
-
}[]
|
| 29 |
-
cameras: {
|
| 30 |
-
frameNum: number
|
| 31 |
-
distance: number
|
| 32 |
-
position: number[]
|
| 33 |
-
rotation: number[]
|
| 34 |
-
interpolation: number[]
|
| 35 |
-
fov: number
|
| 36 |
-
perspective: number
|
| 37 |
-
}[]
|
| 38 |
-
}
|
| 39 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
src/components/friend/frontend/motion-controller.ts
CHANGED
|
@@ -1,8 +1,10 @@
|
|
| 1 |
/**
|
| 2 |
-
* MotionController — unified animation system supporting VRMA
|
| 3 |
*
|
| 4 |
* Uses a single persistent AnimationMixer with crossFade transitions to avoid
|
| 5 |
* T-pose flickering between animations.
|
|
|
|
|
|
|
| 6 |
*/
|
| 7 |
|
| 8 |
import * as THREE from 'three'
|
|
@@ -10,14 +12,11 @@ import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js'
|
|
| 10 |
import { VRMAnimationLoaderPlugin, createVRMAnimationClip } from '@pixiv/three-vrm-animation'
|
| 11 |
import type { VRMAnimation } from '@pixiv/three-vrm-animation'
|
| 12 |
import type { VRM } from '@pixiv/three-vrm'
|
| 13 |
-
import { parseVMDAnimation, bindVMDToVRM } from './vmd-loader'
|
| 14 |
-
import type { VMDAnimationData } from './vmd-loader'
|
| 15 |
import { loadMixamoAnimation } from './mixamo-loader'
|
| 16 |
-
import { VRMIKHandler } from './vrm-ik-handler'
|
| 17 |
|
| 18 |
// ── Motion file types ───────────────────────────────────────────────────────
|
| 19 |
|
| 20 |
-
export type MotionFileType = 'vrma' | '
|
| 21 |
|
| 22 |
// ── Motion presets ──────────────────────────────────────────────────────────
|
| 23 |
|
|
@@ -29,13 +28,11 @@ export interface MotionPreset {
|
|
| 29 |
|
| 30 |
// Actions: short one-shot gestures triggered by emotions / interactions
|
| 31 |
export const actionPresets: Record<string, MotionPreset> = {
|
| 32 |
-
// Local VRMA actions (bundled)
|
| 33 |
akimbo: { label: '叉腰', type: 'vrma', url: '/friend/akimbo.vrma' },
|
| 34 |
playFingers: { label: '搓手', type: 'vrma', url: '/friend/playFingers.vrma' },
|
| 35 |
scratchHead: { label: '挠头', type: 'vrma', url: '/friend/scratchHead.vrma' },
|
| 36 |
stretch: { label: '伸展', type: 'vrma', url: '/friend/stretch.vrma' },
|
| 37 |
|
| 38 |
-
// Local FBX actions (bundled)
|
| 39 |
happy: { label: '开心', type: 'fbx', url: '/friend/happy.fbx' },
|
| 40 |
angry: { label: '生气', type: 'fbx', url: '/friend/angry.fbx' },
|
| 41 |
greeting: { label: '招呼', type: 'fbx', url: '/friend/greeting.fbx' },
|
|
@@ -46,16 +43,6 @@ export const actionPresets: Record<string, MotionPreset> = {
|
|
| 46 |
angryPump: { label: '暴怒', type: 'fbx', url: '/friend/angryPump.fbx' },
|
| 47 |
}
|
| 48 |
|
| 49 |
-
// Dances: looping full-body animations with optional BGM
|
| 50 |
-
export interface DancePreset extends MotionPreset {
|
| 51 |
-
bgm?: string
|
| 52 |
-
}
|
| 53 |
-
|
| 54 |
-
export const dancePresets: Record<string, DancePreset> = {
|
| 55 |
-
jile: { label: '极乐净土', type: 'vmd', url: '/friend/jile.vmd', bgm: '/friend/jile.mp3' },
|
| 56 |
-
love: { label: '恋爱循环', type: 'vmd', url: '/friend/love.vmd', bgm: '/friend/love.mp3' },
|
| 57 |
-
}
|
| 58 |
-
|
| 59 |
// ── Utility: re-anchor root position ────────────────────────────────────────
|
| 60 |
|
| 61 |
function reAnchorRootPositionTrack(clip: THREE.AnimationClip, vrm: VRM) {
|
|
@@ -103,42 +90,44 @@ export class MotionController {
|
|
| 103 |
private idleAction: THREE.AnimationAction | null = null
|
| 104 |
private currentAction: THREE.AnimationAction | null = null
|
| 105 |
private clipCache = new Map<string, THREE.AnimationClip>()
|
| 106 |
-
private vmdDataCache = new Map<string, VMDAnimationData>()
|
| 107 |
private gltfLoader: GLTFLoader
|
| 108 |
-
private ikHandler: VRMIKHandler
|
| 109 |
-
private _isDancing = false
|
| 110 |
private _actionPlaying = false
|
| 111 |
-
private _ikActive = false
|
| 112 |
private holdTimer: ReturnType<typeof setTimeout> | null = null
|
| 113 |
private _actionSafetyTimer: ReturnType<typeof setTimeout> | null = null
|
| 114 |
-
private
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
onDanceStart?: () => void
|
| 118 |
-
onDanceStop?: () => void
|
| 119 |
|
| 120 |
constructor(vrm: VRM) {
|
| 121 |
this.vrm = vrm
|
| 122 |
this.mixer = new THREE.AnimationMixer(vrm.scene)
|
| 123 |
this.gltfLoader = new GLTFLoader()
|
| 124 |
this.gltfLoader.register((parser) => new VRMAnimationLoaderPlugin(parser))
|
| 125 |
-
this.ikHandler = VRMIKHandler.get(vrm)
|
| 126 |
}
|
| 127 |
|
| 128 |
-
private _volume = 0.5
|
| 129 |
-
|
| 130 |
-
get isDancing() { return this._isDancing }
|
| 131 |
get actionPlaying() { return this._actionPlaying }
|
| 132 |
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
|
|
|
|
| 137 |
}
|
| 138 |
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 142 |
}
|
| 143 |
|
| 144 |
// ── CrossFade helper ─────────────────────────────────────────────────────
|
|
@@ -177,25 +166,17 @@ export class MotionController {
|
|
| 177 |
}
|
| 178 |
|
| 179 |
// ── Reset to idle (public) ──────────────────────────────────────────────
|
| 180 |
-
// Full reset: stop BGM + stop dance + restart idle on a fresh mixer.
|
| 181 |
|
| 182 |
resetToIdle() {
|
| 183 |
-
this.stopBgm()
|
| 184 |
this.clearTimers()
|
| 185 |
-
this.disableIK()
|
| 186 |
-
|
| 187 |
-
const wasDancing = this._isDancing
|
| 188 |
-
this._isDancing = false
|
| 189 |
this._actionPlaying = false
|
| 190 |
-
this._actionGeneration++
|
| 191 |
|
| 192 |
// CrossFade back to idle
|
| 193 |
if (this.mixer && this.idleClip) {
|
| 194 |
this.idleAction = this.mixer.clipAction(this.idleClip)
|
| 195 |
this.crossFadeTo(this.idleAction)
|
| 196 |
}
|
| 197 |
-
|
| 198 |
-
if (wasDancing) this.onDanceStop?.()
|
| 199 |
}
|
| 200 |
|
| 201 |
// ── Play a one-shot action ──────────────────────────────────────────────
|
|
@@ -203,15 +184,24 @@ export class MotionController {
|
|
| 203 |
private _actionGeneration = 0
|
| 204 |
|
| 205 |
async playAction(name: string, hold = false) {
|
| 206 |
-
if (this._actionPlaying) return
|
| 207 |
-
if (this._isDancing) return
|
| 208 |
-
|
| 209 |
const preset = actionPresets[name]
|
| 210 |
if (!preset) { console.warn('[Motion] unknown action:', name); return }
|
| 211 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 212 |
// Set lock BEFORE async load to prevent concurrent playAction calls
|
| 213 |
this._actionPlaying = true
|
| 214 |
const gen = ++this._actionGeneration
|
|
|
|
|
|
|
| 215 |
|
| 216 |
const clip = await this.loadClip(preset)
|
| 217 |
|
|
@@ -224,108 +214,51 @@ export class MotionController {
|
|
| 224 |
|
| 225 |
const action = this.mixer.clipAction(clip)
|
| 226 |
action.setLoop(THREE.LoopOnce, 1)
|
| 227 |
-
action.clampWhenFinished = true
|
| 228 |
this.crossFadeTo(action)
|
| 229 |
|
| 230 |
-
|
| 231 |
-
|
| 232 |
-
|
| 233 |
-
|
| 234 |
-
this.
|
| 235 |
-
|
| 236 |
-
if (gen !== this._actionGeneration) return
|
| 237 |
-
this.clearTimers()
|
| 238 |
-
if (hold) {
|
| 239 |
-
this.holdTimer = setTimeout(() => {
|
| 240 |
-
if (gen !== this._actionGeneration) return
|
| 241 |
-
this._actionPlaying = false
|
| 242 |
-
this.disableIK()
|
| 243 |
-
this.startIdle()
|
| 244 |
-
}, 10000)
|
| 245 |
-
} else {
|
| 246 |
-
this._actionPlaying = false
|
| 247 |
-
this.disableIK()
|
| 248 |
-
this.startIdle()
|
| 249 |
}
|
| 250 |
-
}
|
| 251 |
-
const onFinished = () => settle()
|
| 252 |
-
this.mixer?.addEventListener('finished', onFinished)
|
| 253 |
-
|
| 254 |
-
// Safety timeout: guarantee _actionPlaying resets even if 'finished' never fires
|
| 255 |
-
const duration = clip.duration > 0 ? clip.duration : 3
|
| 256 |
-
this._actionSafetyTimer = setTimeout(() => settle(), (duration + 1) * 1000)
|
| 257 |
}
|
| 258 |
|
| 259 |
-
/
|
| 260 |
-
|
| 261 |
-
|
| 262 |
-
|
| 263 |
-
|
|
|
|
| 264 |
|
| 265 |
-
|
| 266 |
-
// Accept preset name, URL, or full DancePreset object
|
| 267 |
-
const preset: DancePreset | undefined =
|
| 268 |
-
typeof nameOrPreset === 'object' ? nameOrPreset : dancePresets[nameOrPreset]
|
| 269 |
-
const clip = preset
|
| 270 |
-
? await this.loadClip(preset)
|
| 271 |
-
: await this.loadClipByUrl(nameOrPreset as string)
|
| 272 |
-
|
| 273 |
-
if (!clip) {
|
| 274 |
-
this._isDancing = false
|
| 275 |
-
return
|
| 276 |
-
}
|
| 277 |
|
| 278 |
-
|
|
|
|
|
|
|
| 279 |
this._actionPlaying = false
|
| 280 |
-
|
| 281 |
-
|
| 282 |
-
|
| 283 |
-
// Play BGM if preset has one
|
| 284 |
-
this.stopBgmImmediate()
|
| 285 |
-
if (preset?.bgm) {
|
| 286 |
-
this.bgmAudio = new Audio(preset.bgm)
|
| 287 |
-
this.bgmAudio.loop = true
|
| 288 |
-
this.bgmAudio.volume = this._volume
|
| 289 |
-
this.bgmAudio.play().catch(() => {})
|
| 290 |
-
}
|
| 291 |
-
|
| 292 |
-
if (!this.mixer) { this._isDancing = false; return }
|
| 293 |
-
const danceAction = this.mixer.clipAction(clip)
|
| 294 |
-
danceAction.setLoop(THREE.LoopRepeat, Infinity)
|
| 295 |
-
this.crossFadeTo(danceAction)
|
| 296 |
-
} catch (err) {
|
| 297 |
-
console.error('Failed to start dance:', err)
|
| 298 |
-
this._isDancing = false
|
| 299 |
}
|
| 300 |
-
}
|
| 301 |
-
|
| 302 |
-
/** Stop BGM with fade-out. Safe to call anytime. */
|
| 303 |
-
private stopBgm() {
|
| 304 |
-
if (!this.bgmAudio) return
|
| 305 |
-
const audio = this.bgmAudio
|
| 306 |
-
this.bgmAudio = null
|
| 307 |
-
const fadeInterval = setInterval(() => {
|
| 308 |
-
audio.volume = Math.max(0, audio.volume - 0.1)
|
| 309 |
-
if (audio.volume <= 0) {
|
| 310 |
-
clearInterval(fadeInterval)
|
| 311 |
-
audio.pause()
|
| 312 |
-
}
|
| 313 |
-
}, 50)
|
| 314 |
-
}
|
| 315 |
|
| 316 |
-
|
| 317 |
-
|
| 318 |
-
|
| 319 |
-
|
| 320 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 321 |
}
|
| 322 |
|
| 323 |
/** Cleanup when controller is being destroyed (model reload etc.) */
|
| 324 |
dispose() {
|
| 325 |
-
this.stopBgm()
|
| 326 |
this.clearTimers()
|
| 327 |
-
this.disableIK()
|
| 328 |
-
this._isDancing = false
|
| 329 |
this._actionPlaying = false
|
| 330 |
this._actionGeneration++
|
| 331 |
if (this.mixer) {
|
|
@@ -339,19 +272,7 @@ export class MotionController {
|
|
| 339 |
|
| 340 |
// ── Internal ────────────────────────────────────────────────────────────
|
| 341 |
|
| 342 |
-
private disableIK() {
|
| 343 |
-
if (this._ikActive) {
|
| 344 |
-
this._ikActive = false
|
| 345 |
-
this.ikHandler.disableAll()
|
| 346 |
-
}
|
| 347 |
-
}
|
| 348 |
-
|
| 349 |
private async loadClip(preset: MotionPreset): Promise<THREE.AnimationClip | null> {
|
| 350 |
-
// For VMD, always rebuild clip with IK binding (IK targets are per-play)
|
| 351 |
-
if (preset.type === 'vmd') {
|
| 352 |
-
return this.loadVMDWithIK(preset.url)
|
| 353 |
-
}
|
| 354 |
-
|
| 355 |
const cached = this.clipCache.get(preset.url)
|
| 356 |
if (cached) return cached
|
| 357 |
|
|
@@ -379,32 +300,6 @@ export class MotionController {
|
|
| 379 |
return clip
|
| 380 |
}
|
| 381 |
|
| 382 |
-
private async loadVMDWithIK(url: string): Promise<THREE.AnimationClip | null> {
|
| 383 |
-
try {
|
| 384 |
-
// Cache parsed VMD data (parsing is expensive), but rebuild clip each time
|
| 385 |
-
// because IK binding creates unique target objects per play
|
| 386 |
-
let data = this.vmdDataCache.get(url)
|
| 387 |
-
if (!data) {
|
| 388 |
-
data = await parseVMDAnimation(url, this.vrm)
|
| 389 |
-
this.vmdDataCache.set(url, data)
|
| 390 |
-
}
|
| 391 |
-
const clip = bindVMDToVRM(data, this.vrm, this.ikHandler)
|
| 392 |
-
this._ikActive = true
|
| 393 |
-
return clip
|
| 394 |
-
} catch (err) {
|
| 395 |
-
console.error('Failed to load VMD:', url, err)
|
| 396 |
-
return null
|
| 397 |
-
}
|
| 398 |
-
}
|
| 399 |
-
|
| 400 |
-
private async loadClipByUrl(url: string): Promise<THREE.AnimationClip | null> {
|
| 401 |
-
const ext = url.split('.').pop()?.toLowerCase()
|
| 402 |
-
const type: MotionFileType =
|
| 403 |
-
ext === 'vmd' ? 'vmd' :
|
| 404 |
-
ext === 'fbx' ? 'fbx' : 'vrma'
|
| 405 |
-
return this.loadClip({ label: url, type, url })
|
| 406 |
-
}
|
| 407 |
-
|
| 408 |
private async loadVRMA(url: string): Promise<THREE.AnimationClip | null> {
|
| 409 |
try {
|
| 410 |
const gltf = await this.gltfLoader.loadAsync(url)
|
|
|
|
| 1 |
/**
|
| 2 |
+
* MotionController — unified animation system supporting VRMA and FBX actions.
|
| 3 |
*
|
| 4 |
* Uses a single persistent AnimationMixer with crossFade transitions to avoid
|
| 5 |
* T-pose flickering between animations.
|
| 6 |
+
*
|
| 7 |
+
* Simplified: removed dance system, kept action presets for human-like behavior.
|
| 8 |
*/
|
| 9 |
|
| 10 |
import * as THREE from 'three'
|
|
|
|
| 12 |
import { VRMAnimationLoaderPlugin, createVRMAnimationClip } from '@pixiv/three-vrm-animation'
|
| 13 |
import type { VRMAnimation } from '@pixiv/three-vrm-animation'
|
| 14 |
import type { VRM } from '@pixiv/three-vrm'
|
|
|
|
|
|
|
| 15 |
import { loadMixamoAnimation } from './mixamo-loader'
|
|
|
|
| 16 |
|
| 17 |
// ── Motion file types ───────────────────────────────────────────────────────
|
| 18 |
|
| 19 |
+
export type MotionFileType = 'vrma' | 'fbx'
|
| 20 |
|
| 21 |
// ── Motion presets ──────────────────────────────────────────────────────────
|
| 22 |
|
|
|
|
| 28 |
|
| 29 |
// Actions: short one-shot gestures triggered by emotions / interactions
|
| 30 |
export const actionPresets: Record<string, MotionPreset> = {
|
|
|
|
| 31 |
akimbo: { label: '叉腰', type: 'vrma', url: '/friend/akimbo.vrma' },
|
| 32 |
playFingers: { label: '搓手', type: 'vrma', url: '/friend/playFingers.vrma' },
|
| 33 |
scratchHead: { label: '挠头', type: 'vrma', url: '/friend/scratchHead.vrma' },
|
| 34 |
stretch: { label: '伸展', type: 'vrma', url: '/friend/stretch.vrma' },
|
| 35 |
|
|
|
|
| 36 |
happy: { label: '开心', type: 'fbx', url: '/friend/happy.fbx' },
|
| 37 |
angry: { label: '生气', type: 'fbx', url: '/friend/angry.fbx' },
|
| 38 |
greeting: { label: '招呼', type: 'fbx', url: '/friend/greeting.fbx' },
|
|
|
|
| 43 |
angryPump: { label: '暴怒', type: 'fbx', url: '/friend/angryPump.fbx' },
|
| 44 |
}
|
| 45 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 46 |
// ── Utility: re-anchor root position ────────────────────────────────────────
|
| 47 |
|
| 48 |
function reAnchorRootPositionTrack(clip: THREE.AnimationClip, vrm: VRM) {
|
|
|
|
| 90 |
private idleAction: THREE.AnimationAction | null = null
|
| 91 |
private currentAction: THREE.AnimationAction | null = null
|
| 92 |
private clipCache = new Map<string, THREE.AnimationClip>()
|
|
|
|
| 93 |
private gltfLoader: GLTFLoader
|
|
|
|
|
|
|
| 94 |
private _actionPlaying = false
|
|
|
|
| 95 |
private holdTimer: ReturnType<typeof setTimeout> | null = null
|
| 96 |
private _actionSafetyTimer: ReturnType<typeof setTimeout> | null = null
|
| 97 |
+
private actionQueue: Array<{ name: string; hold: boolean }> = []
|
| 98 |
+
private _settleGen = 0 // generation at which the current settle is valid
|
| 99 |
+
private _settleHold = false // hold flag for current settle
|
|
|
|
|
|
|
| 100 |
|
| 101 |
constructor(vrm: VRM) {
|
| 102 |
this.vrm = vrm
|
| 103 |
this.mixer = new THREE.AnimationMixer(vrm.scene)
|
| 104 |
this.gltfLoader = new GLTFLoader()
|
| 105 |
this.gltfLoader.register((parser) => new VRMAnimationLoaderPlugin(parser))
|
|
|
|
| 106 |
}
|
| 107 |
|
|
|
|
|
|
|
|
|
|
| 108 |
get actionPlaying() { return this._actionPlaying }
|
| 109 |
|
| 110 |
+
update(delta: number) {
|
| 111 |
+
if (this.mixer) {
|
| 112 |
+
this.mixer.update(delta)
|
| 113 |
+
this.checkActionCompletion()
|
| 114 |
+
}
|
| 115 |
}
|
| 116 |
|
| 117 |
+
/**
|
| 118 |
+
* Frame-accurate action completion detection.
|
| 119 |
+
* THREE.AnimationMixer 'finished' event only fires when ALL actions finish,
|
| 120 |
+
* which never happens with a looping idle. So we check per-frame instead.
|
| 121 |
+
*/
|
| 122 |
+
private checkActionCompletion() {
|
| 123 |
+
if (!this._actionPlaying || !this.currentAction) return
|
| 124 |
+
const clip = this.currentAction.getClip()
|
| 125 |
+
if (!clip || clip.duration <= 0) return
|
| 126 |
+
|
| 127 |
+
// Allow a 1-frame epsilon (≈16ms at 60fps) to avoid precision issues
|
| 128 |
+
if (this.currentAction.time >= clip.duration - 0.02) {
|
| 129 |
+
this.finishCurrentAction()
|
| 130 |
+
}
|
| 131 |
}
|
| 132 |
|
| 133 |
// ── CrossFade helper ─────────────────────────────────────────────────────
|
|
|
|
| 166 |
}
|
| 167 |
|
| 168 |
// ── Reset to idle (public) ──────────────────────────────────────────────
|
|
|
|
| 169 |
|
| 170 |
resetToIdle() {
|
|
|
|
| 171 |
this.clearTimers()
|
|
|
|
|
|
|
|
|
|
|
|
|
| 172 |
this._actionPlaying = false
|
| 173 |
+
this._actionGeneration++
|
| 174 |
|
| 175 |
// CrossFade back to idle
|
| 176 |
if (this.mixer && this.idleClip) {
|
| 177 |
this.idleAction = this.mixer.clipAction(this.idleClip)
|
| 178 |
this.crossFadeTo(this.idleAction)
|
| 179 |
}
|
|
|
|
|
|
|
| 180 |
}
|
| 181 |
|
| 182 |
// ── Play a one-shot action ──────────────────────────────────────────────
|
|
|
|
| 184 |
private _actionGeneration = 0
|
| 185 |
|
| 186 |
async playAction(name: string, hold = false) {
|
|
|
|
|
|
|
|
|
|
| 187 |
const preset = actionPresets[name]
|
| 188 |
if (!preset) { console.warn('[Motion] unknown action:', name); return }
|
| 189 |
|
| 190 |
+
// If already playing an action, queue instead of dropping
|
| 191 |
+
if (this._actionPlaying) {
|
| 192 |
+
// Avoid duplicate consecutive queued items
|
| 193 |
+
const last = this.actionQueue[this.actionQueue.length - 1]
|
| 194 |
+
if (!last || last.name !== name) {
|
| 195 |
+
this.actionQueue.push({ name, hold })
|
| 196 |
+
}
|
| 197 |
+
return
|
| 198 |
+
}
|
| 199 |
+
|
| 200 |
// Set lock BEFORE async load to prevent concurrent playAction calls
|
| 201 |
this._actionPlaying = true
|
| 202 |
const gen = ++this._actionGeneration
|
| 203 |
+
this._settleGen = gen
|
| 204 |
+
this._settleHold = hold
|
| 205 |
|
| 206 |
const clip = await this.loadClip(preset)
|
| 207 |
|
|
|
|
| 214 |
|
| 215 |
const action = this.mixer.clipAction(clip)
|
| 216 |
action.setLoop(THREE.LoopOnce, 1)
|
| 217 |
+
action.clampWhenFinished = true
|
| 218 |
this.crossFadeTo(action)
|
| 219 |
|
| 220 |
+
// Frame-accurate completion is handled in checkActionCompletion() via update()
|
| 221 |
+
// Safety timer: generous fallback (should never fire if checkActionCompletion works)
|
| 222 |
+
const safeDuration = Math.max(clip.duration, 3) + 5
|
| 223 |
+
this._actionSafetyTimer = setTimeout(() => {
|
| 224 |
+
if (gen === this._actionGeneration && this._actionPlaying) {
|
| 225 |
+
this.finishCurrentAction()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 226 |
}
|
| 227 |
+
}, safeDuration * 1000)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 228 |
}
|
| 229 |
|
| 230 |
+
/** Called by checkActionCompletion() or safety timer when current action ends. */
|
| 231 |
+
private finishCurrentAction() {
|
| 232 |
+
if (!this._actionPlaying) return
|
| 233 |
+
this.clearTimers()
|
| 234 |
+
const gen = this._settleGen
|
| 235 |
+
const hold = this._settleHold
|
| 236 |
|
| 237 |
+
if (gen !== this._actionGeneration) return
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 238 |
|
| 239 |
+
// Process next queued action
|
| 240 |
+
const next = this.actionQueue.shift()
|
| 241 |
+
if (next) {
|
| 242 |
this._actionPlaying = false
|
| 243 |
+
this.playAction(next.name, next.hold)
|
| 244 |
+
return
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 245 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 246 |
|
| 247 |
+
if (hold) {
|
| 248 |
+
this.holdTimer = setTimeout(() => {
|
| 249 |
+
if (gen !== this._actionGeneration) return
|
| 250 |
+
this._actionPlaying = false
|
| 251 |
+
this.startIdle()
|
| 252 |
+
}, 10000)
|
| 253 |
+
} else {
|
| 254 |
+
this._actionPlaying = false
|
| 255 |
+
this.startIdle()
|
| 256 |
+
}
|
| 257 |
}
|
| 258 |
|
| 259 |
/** Cleanup when controller is being destroyed (model reload etc.) */
|
| 260 |
dispose() {
|
|
|
|
| 261 |
this.clearTimers()
|
|
|
|
|
|
|
| 262 |
this._actionPlaying = false
|
| 263 |
this._actionGeneration++
|
| 264 |
if (this.mixer) {
|
|
|
|
| 272 |
|
| 273 |
// ── Internal ────────────────────────────────────────────────────────────
|
| 274 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 275 |
private async loadClip(preset: MotionPreset): Promise<THREE.AnimationClip | null> {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 276 |
const cached = this.clipCache.get(preset.url)
|
| 277 |
if (cached) return cached
|
| 278 |
|
|
|
|
| 300 |
return clip
|
| 301 |
}
|
| 302 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 303 |
private async loadVRMA(url: string): Promise<THREE.AnimationClip | null> {
|
| 304 |
try {
|
| 305 |
const gltf = await this.gltfLoader.loadAsync(url)
|
src/components/friend/frontend/vmd-loader.ts
DELETED
|
@@ -1,663 +0,0 @@
|
|
| 1 |
-
/**
|
| 2 |
-
* VMD animation loader for VRM models.
|
| 3 |
-
* Ported from lobe-vidol's VMDAnimation module.
|
| 4 |
-
*
|
| 5 |
-
* Parses VMD binary files (via mmd-parser), converts bone/morph keyframes
|
| 6 |
-
* to VRM-compatible AnimationClips with IK support.
|
| 7 |
-
*/
|
| 8 |
-
|
| 9 |
-
import {
|
| 10 |
-
VRMExpressionPresetName as BlendShapePresetName,
|
| 11 |
-
VRMHumanBoneName as HumanoidBoneName,
|
| 12 |
-
} from '@pixiv/three-vrm-core'
|
| 13 |
-
import type { VRM, VRMHumanBoneName } from '@pixiv/three-vrm'
|
| 14 |
-
import { CharsetEncoder, Parser } from 'mmd-parser'
|
| 15 |
-
import type { VmdFile } from 'mmd-parser'
|
| 16 |
-
import {
|
| 17 |
-
AnimationClip,
|
| 18 |
-
KeyframeTrack,
|
| 19 |
-
MathUtils,
|
| 20 |
-
NumberKeyframeTrack,
|
| 21 |
-
Object3D,
|
| 22 |
-
Quaternion,
|
| 23 |
-
QuaternionKeyframeTrack,
|
| 24 |
-
Vector3,
|
| 25 |
-
VectorKeyframeTrack,
|
| 26 |
-
} from 'three'
|
| 27 |
-
|
| 28 |
-
import { VRMIKHandler } from './vrm-ik-handler'
|
| 29 |
-
|
| 30 |
-
// ── Types ───────────────────────────────────────────────────────────────────
|
| 31 |
-
|
| 32 |
-
export interface VMDAnimationData {
|
| 33 |
-
duration: number
|
| 34 |
-
timelines: Timeline[]
|
| 35 |
-
}
|
| 36 |
-
|
| 37 |
-
interface Timeline {
|
| 38 |
-
name: VRMHumanBoneName | BlendShapePresetName
|
| 39 |
-
type: 'position' | 'rotation' | 'morph'
|
| 40 |
-
isIK?: boolean
|
| 41 |
-
times: number[]
|
| 42 |
-
values: number[]
|
| 43 |
-
}
|
| 44 |
-
|
| 45 |
-
interface VRMOffsets {
|
| 46 |
-
hipsOffset?: number[]
|
| 47 |
-
leftFootOffset?: number[]
|
| 48 |
-
leftToeOffset?: number[]
|
| 49 |
-
rightFootOffset?: number[]
|
| 50 |
-
rightToeOffset?: number[]
|
| 51 |
-
}
|
| 52 |
-
|
| 53 |
-
interface Keyframe {
|
| 54 |
-
boneName: string
|
| 55 |
-
frameNum: number
|
| 56 |
-
position: Vector3
|
| 57 |
-
rotation: Quaternion
|
| 58 |
-
}
|
| 59 |
-
|
| 60 |
-
interface LerpKeyframe extends Keyframe {
|
| 61 |
-
isNew?: boolean
|
| 62 |
-
}
|
| 63 |
-
|
| 64 |
-
interface IKOffsetInit {
|
| 65 |
-
x: number
|
| 66 |
-
y: number
|
| 67 |
-
z: number
|
| 68 |
-
s?: number
|
| 69 |
-
sx?: number
|
| 70 |
-
sy?: number
|
| 71 |
-
sz?: number
|
| 72 |
-
o?: number
|
| 73 |
-
ox?: number
|
| 74 |
-
oy?: number
|
| 75 |
-
oz?: number
|
| 76 |
-
dx?: boolean
|
| 77 |
-
dy?: boolean
|
| 78 |
-
dz?: boolean
|
| 79 |
-
}
|
| 80 |
-
|
| 81 |
-
// ── Hot-patch CharsetEncoder to forgive invalid charcodes ───────────────────
|
| 82 |
-
|
| 83 |
-
{
|
| 84 |
-
const { s2u } = CharsetEncoder.prototype
|
| 85 |
-
CharsetEncoder.prototype.s2u = function (array: any) {
|
| 86 |
-
try {
|
| 87 |
-
return s2u.call(this, array)
|
| 88 |
-
} catch (e) {
|
| 89 |
-
if (typeof e === 'string') {
|
| 90 |
-
console.warn(`Charset encoder: ${e}`)
|
| 91 |
-
return ''
|
| 92 |
-
}
|
| 93 |
-
throw e
|
| 94 |
-
}
|
| 95 |
-
}
|
| 96 |
-
}
|
| 97 |
-
|
| 98 |
-
// ── Constants ───────────────────────────────────────────────────────────────
|
| 99 |
-
|
| 100 |
-
function isTruely<T>(x: T): x is Exclude<T, false | '' | 0 | null | undefined> {
|
| 101 |
-
return !!x
|
| 102 |
-
}
|
| 103 |
-
|
| 104 |
-
const tempV3 = new Vector3()
|
| 105 |
-
const tempQ = new Quaternion()
|
| 106 |
-
|
| 107 |
-
const VMDBoneNames = {
|
| 108 |
-
Root: '全ての親',
|
| 109 |
-
Center: 'センター',
|
| 110 |
-
Hips: '下半身',
|
| 111 |
-
Spine: '上半身',
|
| 112 |
-
Chest: '上半身2',
|
| 113 |
-
Neck: '首',
|
| 114 |
-
Head: '頭',
|
| 115 |
-
LeftEye: '左目',
|
| 116 |
-
LeftShoulder: '左肩',
|
| 117 |
-
LeftUpperArm: '左腕',
|
| 118 |
-
LeftLowerArm: '左ひじ',
|
| 119 |
-
LeftHand: '左手首',
|
| 120 |
-
LeftThumbProximal: '左親指0',
|
| 121 |
-
LeftThumbIntermediate: '左親指1',
|
| 122 |
-
LeftThumbDistal: '左親指2',
|
| 123 |
-
LeftIndexProximal: '左人指1',
|
| 124 |
-
LeftIndexIntermediate: '左人指2',
|
| 125 |
-
LeftIndexDistal: '左人指3',
|
| 126 |
-
LeftMiddleProximal: '左中指1',
|
| 127 |
-
LeftMiddleIntermediate: '左中指2',
|
| 128 |
-
LeftMiddleDistal: '左中指3',
|
| 129 |
-
LeftRingProximal: '左薬指1',
|
| 130 |
-
LeftRingIntermediate: '左薬指2',
|
| 131 |
-
LeftRingDistal: '左薬指3',
|
| 132 |
-
LeftLittleProximal: '左小指1',
|
| 133 |
-
LeftLittleIntermediate: '左小指2',
|
| 134 |
-
LeftLittleDistal: '左小指3',
|
| 135 |
-
LeftUpperLeg: '左足',
|
| 136 |
-
LeftLowerLeg: '左ひざ',
|
| 137 |
-
LeftFoot: '左足首',
|
| 138 |
-
LeftFootIK: '左足IK',
|
| 139 |
-
LeftToes: '左つま先',
|
| 140 |
-
LeftToeIK: '左つま先IK',
|
| 141 |
-
RightEye: '右目',
|
| 142 |
-
RightShoulder: '右肩',
|
| 143 |
-
RightUpperArm: '右腕',
|
| 144 |
-
RightLowerArm: '右ひじ',
|
| 145 |
-
RightHand: '右手首',
|
| 146 |
-
RightThumbProximal: '右親指0',
|
| 147 |
-
RightThumbIntermediate: '右親指1',
|
| 148 |
-
RightThumbDistal: '右親指2',
|
| 149 |
-
RightIndexProximal: '右人指1',
|
| 150 |
-
RightIndexIntermediate: '右人指2',
|
| 151 |
-
RightIndexDistal: '右人指3',
|
| 152 |
-
RightMiddleProximal: '右中指1',
|
| 153 |
-
RightMiddleIntermediate: '右中指2',
|
| 154 |
-
RightMiddleDistal: '右中指3',
|
| 155 |
-
RightRingProximal: '右薬指1',
|
| 156 |
-
RightRingIntermediate: '右薬指2',
|
| 157 |
-
RightRingDistal: '右薬指3',
|
| 158 |
-
RightLittleProximal: '右小指1',
|
| 159 |
-
RightLittleIntermediate: '右小指2',
|
| 160 |
-
RightLittleDistal: '右小指3',
|
| 161 |
-
RightUpperLeg: '右足',
|
| 162 |
-
RightLowerLeg: '右ひざ',
|
| 163 |
-
RightFoot: '右足首',
|
| 164 |
-
RightFootIK: '右足IK',
|
| 165 |
-
RightToes: '右つま先',
|
| 166 |
-
RightToeIK: '右つま先IK',
|
| 167 |
-
} as const
|
| 168 |
-
|
| 169 |
-
const VMDMorphNames = {
|
| 170 |
-
Blink: 'まばたき',
|
| 171 |
-
BlinkR: 'ウィンク',
|
| 172 |
-
BlinkL: 'ウィンク右',
|
| 173 |
-
A: 'あ',
|
| 174 |
-
I: 'い',
|
| 175 |
-
U: 'う',
|
| 176 |
-
E: 'え',
|
| 177 |
-
O: 'お',
|
| 178 |
-
} as const
|
| 179 |
-
|
| 180 |
-
const VRM_VMD_BONE_MAP = new Map<string, HumanoidBoneName>([
|
| 181 |
-
[VMDBoneNames.Hips, HumanoidBoneName.Hips],
|
| 182 |
-
[VMDBoneNames.Spine, HumanoidBoneName.Spine],
|
| 183 |
-
[VMDBoneNames.Chest, HumanoidBoneName.Chest],
|
| 184 |
-
[VMDBoneNames.Neck, HumanoidBoneName.Neck],
|
| 185 |
-
[VMDBoneNames.Head, HumanoidBoneName.Head],
|
| 186 |
-
[VMDBoneNames.LeftEye, HumanoidBoneName.LeftEye],
|
| 187 |
-
[VMDBoneNames.LeftShoulder, HumanoidBoneName.LeftShoulder],
|
| 188 |
-
[VMDBoneNames.LeftUpperArm, HumanoidBoneName.LeftUpperArm],
|
| 189 |
-
[VMDBoneNames.LeftLowerArm, HumanoidBoneName.LeftLowerArm],
|
| 190 |
-
[VMDBoneNames.LeftHand, HumanoidBoneName.LeftHand],
|
| 191 |
-
[VMDBoneNames.LeftThumbProximal, HumanoidBoneName.LeftThumbProximal],
|
| 192 |
-
[VMDBoneNames.LeftThumbIntermediate, HumanoidBoneName.LeftThumbMetacarpal],
|
| 193 |
-
[VMDBoneNames.LeftThumbDistal, HumanoidBoneName.LeftThumbDistal],
|
| 194 |
-
[VMDBoneNames.LeftIndexProximal, HumanoidBoneName.LeftIndexProximal],
|
| 195 |
-
[VMDBoneNames.LeftIndexIntermediate, HumanoidBoneName.LeftIndexIntermediate],
|
| 196 |
-
[VMDBoneNames.LeftIndexDistal, HumanoidBoneName.LeftIndexDistal],
|
| 197 |
-
[VMDBoneNames.LeftMiddleProximal, HumanoidBoneName.LeftMiddleProximal],
|
| 198 |
-
[VMDBoneNames.LeftMiddleIntermediate, HumanoidBoneName.LeftMiddleIntermediate],
|
| 199 |
-
[VMDBoneNames.LeftMiddleDistal, HumanoidBoneName.LeftMiddleDistal],
|
| 200 |
-
[VMDBoneNames.LeftRingProximal, HumanoidBoneName.LeftRingProximal],
|
| 201 |
-
[VMDBoneNames.LeftRingIntermediate, HumanoidBoneName.LeftRingIntermediate],
|
| 202 |
-
[VMDBoneNames.LeftRingDistal, HumanoidBoneName.LeftRingDistal],
|
| 203 |
-
[VMDBoneNames.LeftLittleProximal, HumanoidBoneName.LeftLittleProximal],
|
| 204 |
-
[VMDBoneNames.LeftLittleIntermediate, HumanoidBoneName.LeftLittleIntermediate],
|
| 205 |
-
[VMDBoneNames.LeftLittleDistal, HumanoidBoneName.LeftLittleDistal],
|
| 206 |
-
[VMDBoneNames.LeftUpperLeg, HumanoidBoneName.LeftUpperLeg],
|
| 207 |
-
[VMDBoneNames.LeftLowerLeg, HumanoidBoneName.LeftLowerLeg],
|
| 208 |
-
[VMDBoneNames.LeftFoot, HumanoidBoneName.LeftFoot],
|
| 209 |
-
[VMDBoneNames.LeftToes, HumanoidBoneName.LeftToes],
|
| 210 |
-
[VMDBoneNames.RightEye, HumanoidBoneName.RightEye],
|
| 211 |
-
[VMDBoneNames.RightShoulder, HumanoidBoneName.RightShoulder],
|
| 212 |
-
[VMDBoneNames.RightUpperArm, HumanoidBoneName.RightUpperArm],
|
| 213 |
-
[VMDBoneNames.RightLowerArm, HumanoidBoneName.RightLowerArm],
|
| 214 |
-
[VMDBoneNames.RightHand, HumanoidBoneName.RightHand],
|
| 215 |
-
[VMDBoneNames.RightThumbProximal, HumanoidBoneName.RightThumbProximal],
|
| 216 |
-
[VMDBoneNames.RightThumbIntermediate, HumanoidBoneName.RightThumbMetacarpal],
|
| 217 |
-
[VMDBoneNames.RightThumbDistal, HumanoidBoneName.RightThumbDistal],
|
| 218 |
-
[VMDBoneNames.RightIndexProximal, HumanoidBoneName.RightIndexProximal],
|
| 219 |
-
[VMDBoneNames.RightIndexIntermediate, HumanoidBoneName.RightIndexIntermediate],
|
| 220 |
-
[VMDBoneNames.RightIndexDistal, HumanoidBoneName.RightIndexDistal],
|
| 221 |
-
[VMDBoneNames.RightMiddleProximal, HumanoidBoneName.RightMiddleProximal],
|
| 222 |
-
[VMDBoneNames.RightMiddleIntermediate, HumanoidBoneName.RightMiddleIntermediate],
|
| 223 |
-
[VMDBoneNames.RightMiddleDistal, HumanoidBoneName.RightMiddleDistal],
|
| 224 |
-
[VMDBoneNames.RightRingProximal, HumanoidBoneName.RightRingProximal],
|
| 225 |
-
[VMDBoneNames.RightRingIntermediate, HumanoidBoneName.RightRingIntermediate],
|
| 226 |
-
[VMDBoneNames.RightRingDistal, HumanoidBoneName.RightRingDistal],
|
| 227 |
-
[VMDBoneNames.RightLittleProximal, HumanoidBoneName.RightLittleProximal],
|
| 228 |
-
[VMDBoneNames.RightLittleIntermediate, HumanoidBoneName.RightLittleIntermediate],
|
| 229 |
-
[VMDBoneNames.RightLittleDistal, HumanoidBoneName.RightLittleDistal],
|
| 230 |
-
[VMDBoneNames.RightUpperLeg, HumanoidBoneName.RightUpperLeg],
|
| 231 |
-
[VMDBoneNames.RightLowerLeg, HumanoidBoneName.RightLowerLeg],
|
| 232 |
-
[VMDBoneNames.RightFoot, HumanoidBoneName.RightFoot],
|
| 233 |
-
[VMDBoneNames.RightToes, HumanoidBoneName.RightToes],
|
| 234 |
-
])
|
| 235 |
-
|
| 236 |
-
const VMD_VRM_IK_MAP = new Map<string, HumanoidBoneName>([
|
| 237 |
-
[VMDBoneNames.LeftFootIK, HumanoidBoneName.LeftFoot],
|
| 238 |
-
[VMDBoneNames.LeftToeIK, HumanoidBoneName.LeftToes],
|
| 239 |
-
[VMDBoneNames.RightFootIK, HumanoidBoneName.RightFoot],
|
| 240 |
-
[VMDBoneNames.RightToeIK, HumanoidBoneName.RightToes],
|
| 241 |
-
])
|
| 242 |
-
|
| 243 |
-
const VMD_BONE_NAMES = new Set<string>(VRM_VMD_BONE_MAP.keys())
|
| 244 |
-
Array.from(VMD_VRM_IK_MAP.keys()).forEach(VMD_BONE_NAMES.add, VMD_BONE_NAMES)
|
| 245 |
-
VMD_BONE_NAMES.add(VMDBoneNames.Root)
|
| 246 |
-
VMD_BONE_NAMES.add(VMDBoneNames.Center)
|
| 247 |
-
|
| 248 |
-
const VMD_VRM_MORPH_MAP = new Map<string, BlendShapePresetName>([
|
| 249 |
-
[VMDMorphNames.Blink, BlendShapePresetName.Blink],
|
| 250 |
-
[VMDMorphNames.BlinkL, BlendShapePresetName.BlinkLeft],
|
| 251 |
-
[VMDMorphNames.BlinkR, BlendShapePresetName.BlinkRight],
|
| 252 |
-
[VMDMorphNames.A, BlendShapePresetName.Aa],
|
| 253 |
-
[VMDMorphNames.I, BlendShapePresetName.Ih],
|
| 254 |
-
[VMDMorphNames.U, BlendShapePresetName.Ou],
|
| 255 |
-
[VMDMorphNames.E, BlendShapePresetName.Ee],
|
| 256 |
-
[VMDMorphNames.O, BlendShapePresetName.Oh],
|
| 257 |
-
])
|
| 258 |
-
|
| 259 |
-
const IK_OFFSET_INIT = new Map<string, IKOffsetInit>([
|
| 260 |
-
[VMDBoneNames.Center, { x: 0, y: 1, z: 0, s: 10 }],
|
| 261 |
-
[VMDBoneNames.LeftFootIK, { x: 1, y: 1, z: 0, s: 10, dx: true }],
|
| 262 |
-
[VMDBoneNames.RightFootIK, { x: -1, y: 1, z: 0, s: 10, dx: true }],
|
| 263 |
-
[VMDBoneNames.LeftToeIK, { x: 0, y: -1, z: -1, s: 10, oy: 2.5, dx: true, dz: true }],
|
| 264 |
-
[VMDBoneNames.RightToeIK, { x: 0, y: -1, z: -1, s: 10, oy: 2.5, dx: true, dz: true }],
|
| 265 |
-
])
|
| 266 |
-
|
| 267 |
-
const V3_ZERO = new Vector3()
|
| 268 |
-
const Q_IDENTITY = new Quaternion()
|
| 269 |
-
const Z_30_DEG_CW = new Quaternion().setFromAxisAngle(tempV3.set(0, 0, 1), 30 * MathUtils.DEG2RAD)
|
| 270 |
-
const Z_30_DEG_CCW = Z_30_DEG_CW.clone().invert()
|
| 271 |
-
|
| 272 |
-
// ── VMD Parsing ────────��────────────────────────────────────────────────────
|
| 273 |
-
|
| 274 |
-
function convertVMD(buffer: ArrayBufferLike, vrmOffset?: VRMOffsets): VMDAnimationData {
|
| 275 |
-
const vmd = new Parser().parseVmd(buffer)
|
| 276 |
-
const morphs = convertMorphs(vmd)
|
| 277 |
-
const motions = convertMotions(vmd, vrmOffset)
|
| 278 |
-
return {
|
| 279 |
-
duration: Math.max(morphs.duration, motions.duration),
|
| 280 |
-
timelines: Array.prototype.concat(morphs.timelines, motions.timelines),
|
| 281 |
-
}
|
| 282 |
-
}
|
| 283 |
-
|
| 284 |
-
function convertMorphs({ morphs }: VmdFile): VMDAnimationData {
|
| 285 |
-
sortFrames(morphs)
|
| 286 |
-
const timelines = new Map<string, Timeline>()
|
| 287 |
-
for (const { morphName, weight, frameNum } of morphs) {
|
| 288 |
-
const name = VMD_VRM_MORPH_MAP.get(morphName)
|
| 289 |
-
if (!name) continue
|
| 290 |
-
let timeline = timelines.get(name)
|
| 291 |
-
if (!timeline)
|
| 292 |
-
timelines.set(name, (timeline = { name, type: 'morph', times: [], values: [] }))
|
| 293 |
-
const { times, values } = timeline
|
| 294 |
-
const time = frameNum / 30
|
| 295 |
-
const timeIndex = times.findIndex((t) => t === time)
|
| 296 |
-
if (timeIndex < 0) {
|
| 297 |
-
times.push(time)
|
| 298 |
-
values.push(weight)
|
| 299 |
-
} else {
|
| 300 |
-
values[timeIndex] = Math.max(values[timeIndex], weight)
|
| 301 |
-
}
|
| 302 |
-
}
|
| 303 |
-
return {
|
| 304 |
-
timelines: Array.from(timelines.values()),
|
| 305 |
-
duration: getLastFrameNum(morphs) / 30,
|
| 306 |
-
}
|
| 307 |
-
}
|
| 308 |
-
|
| 309 |
-
function convertMotions({ motions }: VmdFile, vrmOffset?: VRMOffsets): VMDAnimationData {
|
| 310 |
-
sortFrames(motions)
|
| 311 |
-
const timelines: Timeline[] = []
|
| 312 |
-
const boneTlMap = new Map<string, Keyframe[]>()
|
| 313 |
-
for (const name of VMD_BONE_NAMES) boneTlMap.set(name, [])
|
| 314 |
-
for (const { boneName, frameNum, position, rotation } of motions)
|
| 315 |
-
boneTlMap.get(boneName)?.push({
|
| 316 |
-
boneName,
|
| 317 |
-
frameNum,
|
| 318 |
-
position: new Vector3().fromArray(position),
|
| 319 |
-
rotation: new Quaternion().fromArray(rotation),
|
| 320 |
-
})
|
| 321 |
-
fixPositions(boneTlMap, vrmOffset)
|
| 322 |
-
for (const [boneName, timeline] of boneTlMap) {
|
| 323 |
-
let name = VRM_VMD_BONE_MAP.get(boneName)
|
| 324 |
-
let isIK = false
|
| 325 |
-
if (!name) {
|
| 326 |
-
isIK = VMD_VRM_IK_MAP.has(boneName)
|
| 327 |
-
name = VMD_VRM_IK_MAP.get(boneName)
|
| 328 |
-
}
|
| 329 |
-
if (name) {
|
| 330 |
-
const times: number[] = []
|
| 331 |
-
const positions: number[] = []
|
| 332 |
-
const rotations: number[] = []
|
| 333 |
-
for (const f of timeline) {
|
| 334 |
-
const i = times.push(f.frameNum / 30) - 1
|
| 335 |
-
f.position.toArray(positions, i * 3)
|
| 336 |
-
f.rotation.toArray(rotations, i * 4)
|
| 337 |
-
}
|
| 338 |
-
if (times.length) {
|
| 339 |
-
timelines.push({ name, type: 'rotation', isIK, times, values: rotations })
|
| 340 |
-
if (isIK || name === HumanoidBoneName.Hips)
|
| 341 |
-
timelines.push({ name, type: 'position', isIK, times, values: positions })
|
| 342 |
-
}
|
| 343 |
-
}
|
| 344 |
-
}
|
| 345 |
-
return { timelines, duration: getLastFrameNum(motions) / 30 }
|
| 346 |
-
}
|
| 347 |
-
|
| 348 |
-
function fixPositions(tls: Map<string, Keyframe[]>, vrmOffset: VRMOffsets = {}) {
|
| 349 |
-
const centerOffset = mergeTimelines(
|
| 350 |
-
tls,
|
| 351 |
-
VMDBoneNames.Center,
|
| 352 |
-
offsetToTimeline(VMDBoneNames.Center, vrmOffset.hipsOffset),
|
| 353 |
-
)
|
| 354 |
-
const hipsTl = mergeTimelines(tls, VMDBoneNames.Root, centerOffset, VMDBoneNames.Hips)
|
| 355 |
-
tls.set(
|
| 356 |
-
VMDBoneNames.Spine,
|
| 357 |
-
localizeTimeline(
|
| 358 |
-
hipsTl,
|
| 359 |
-
mergeTimelines(tls, VMDBoneNames.Root, centerOffset, VMDBoneNames.Hips),
|
| 360 |
-
),
|
| 361 |
-
)
|
| 362 |
-
tls.set(VMDBoneNames.Hips, hipsTl)
|
| 363 |
-
const leftFootOffset = offsetToTimeline(VMDBoneNames.LeftFootIK, vrmOffset.leftFootOffset)
|
| 364 |
-
const rightFootOffset = offsetToTimeline(VMDBoneNames.RightFootIK, vrmOffset.rightFootOffset)
|
| 365 |
-
if (tls.has(VMDBoneNames.LeftToeIK))
|
| 366 |
-
tls.set(
|
| 367 |
-
VMDBoneNames.LeftToeIK,
|
| 368 |
-
mergeTimelines(
|
| 369 |
-
tls,
|
| 370 |
-
VMDBoneNames.Root,
|
| 371 |
-
leftFootOffset,
|
| 372 |
-
VMDBoneNames.LeftFootIK,
|
| 373 |
-
offsetToTimeline(VMDBoneNames.RightToeIK, vrmOffset.leftToeOffset),
|
| 374 |
-
VMDBoneNames.LeftToeIK,
|
| 375 |
-
),
|
| 376 |
-
)
|
| 377 |
-
if (tls.has(VMDBoneNames.RightToeIK))
|
| 378 |
-
tls.set(
|
| 379 |
-
VMDBoneNames.RightToeIK,
|
| 380 |
-
mergeTimelines(
|
| 381 |
-
tls,
|
| 382 |
-
VMDBoneNames.Root,
|
| 383 |
-
rightFootOffset,
|
| 384 |
-
VMDBoneNames.RightFootIK,
|
| 385 |
-
offsetToTimeline(VMDBoneNames.RightToeIK, vrmOffset.rightToeOffset),
|
| 386 |
-
VMDBoneNames.RightToeIK,
|
| 387 |
-
),
|
| 388 |
-
)
|
| 389 |
-
if (tls.has(VMDBoneNames.LeftFootIK))
|
| 390 |
-
tls.set(
|
| 391 |
-
VMDBoneNames.LeftFootIK,
|
| 392 |
-
mergeTimelines(tls, VMDBoneNames.Root, leftFootOffset, VMDBoneNames.LeftFootIK),
|
| 393 |
-
)
|
| 394 |
-
if (tls.has(VMDBoneNames.RightFootIK))
|
| 395 |
-
tls.set(
|
| 396 |
-
VMDBoneNames.RightFootIK,
|
| 397 |
-
mergeTimelines(tls, VMDBoneNames.Root, rightFootOffset, VMDBoneNames.RightFootIK),
|
| 398 |
-
)
|
| 399 |
-
tls.delete(VMDBoneNames.Center)
|
| 400 |
-
tls.delete(VMDBoneNames.Root)
|
| 401 |
-
for (const tl of tls.values())
|
| 402 |
-
for (const f of tl) {
|
| 403 |
-
f.position.x *= -1
|
| 404 |
-
f.rotation.x *= -1
|
| 405 |
-
f.rotation.w *= -1
|
| 406 |
-
switch (f.boneName) {
|
| 407 |
-
case VMDBoneNames.LeftUpperArm:
|
| 408 |
-
f.rotation.multiply(Z_30_DEG_CW)
|
| 409 |
-
break
|
| 410 |
-
case VMDBoneNames.RightUpperArm:
|
| 411 |
-
f.rotation.multiply(Z_30_DEG_CCW)
|
| 412 |
-
break
|
| 413 |
-
case VMDBoneNames.LeftLowerArm:
|
| 414 |
-
case VMDBoneNames.LeftHand:
|
| 415 |
-
case VMDBoneNames.LeftThumbProximal:
|
| 416 |
-
case VMDBoneNames.LeftThumbIntermediate:
|
| 417 |
-
case VMDBoneNames.LeftThumbDistal:
|
| 418 |
-
case VMDBoneNames.LeftIndexProximal:
|
| 419 |
-
case VMDBoneNames.LeftIndexIntermediate:
|
| 420 |
-
case VMDBoneNames.LeftIndexDistal:
|
| 421 |
-
case VMDBoneNames.LeftMiddleProximal:
|
| 422 |
-
case VMDBoneNames.LeftMiddleIntermediate:
|
| 423 |
-
case VMDBoneNames.LeftMiddleDistal:
|
| 424 |
-
case VMDBoneNames.LeftRingProximal:
|
| 425 |
-
case VMDBoneNames.LeftRingIntermediate:
|
| 426 |
-
case VMDBoneNames.LeftRingDistal:
|
| 427 |
-
case VMDBoneNames.LeftLittleProximal:
|
| 428 |
-
case VMDBoneNames.LeftLittleIntermediate:
|
| 429 |
-
case VMDBoneNames.LeftLittleDistal:
|
| 430 |
-
f.rotation.premultiply(Z_30_DEG_CCW).multiply(Z_30_DEG_CW)
|
| 431 |
-
break
|
| 432 |
-
case VMDBoneNames.RightLowerArm:
|
| 433 |
-
case VMDBoneNames.RightHand:
|
| 434 |
-
case VMDBoneNames.RightThumbProximal:
|
| 435 |
-
case VMDBoneNames.RightThumbIntermediate:
|
| 436 |
-
case VMDBoneNames.RightThumbDistal:
|
| 437 |
-
case VMDBoneNames.RightIndexProximal:
|
| 438 |
-
case VMDBoneNames.RightIndexIntermediate:
|
| 439 |
-
case VMDBoneNames.RightIndexDistal:
|
| 440 |
-
case VMDBoneNames.RightMiddleProximal:
|
| 441 |
-
case VMDBoneNames.RightMiddleIntermediate:
|
| 442 |
-
case VMDBoneNames.RightMiddleDistal:
|
| 443 |
-
case VMDBoneNames.RightRingProximal:
|
| 444 |
-
case VMDBoneNames.RightRingIntermediate:
|
| 445 |
-
case VMDBoneNames.RightRingDistal:
|
| 446 |
-
case VMDBoneNames.RightLittleProximal:
|
| 447 |
-
case VMDBoneNames.RightLittleIntermediate:
|
| 448 |
-
case VMDBoneNames.RightLittleDistal:
|
| 449 |
-
f.rotation.premultiply(Z_30_DEG_CW).multiply(Z_30_DEG_CCW)
|
| 450 |
-
break
|
| 451 |
-
}
|
| 452 |
-
f.position.multiplyScalar(0.1)
|
| 453 |
-
}
|
| 454 |
-
}
|
| 455 |
-
|
| 456 |
-
function offsetToTimeline(boneName: string, rawPos: number[] | undefined): Keyframe[] {
|
| 457 |
-
const init = IK_OFFSET_INIT.get(boneName)!
|
| 458 |
-
return [
|
| 459 |
-
{
|
| 460 |
-
boneName: `${boneName}Offset`,
|
| 461 |
-
frameNum: 0,
|
| 462 |
-
position: rawPos
|
| 463 |
-
? new Vector3(
|
| 464 |
-
init.dx || isNaN(rawPos[0])
|
| 465 |
-
? init.x
|
| 466 |
-
: rawPos[0] * (init.sx ?? 1) * (init.s ?? 1) + (init.ox ?? 0) + (init.o ?? 0),
|
| 467 |
-
init.dy || isNaN(rawPos[1])
|
| 468 |
-
? init.y
|
| 469 |
-
: rawPos[1] * (init.sy ?? 1) * (init.s ?? 1) + (init.oy ?? 0) + (init.o ?? 0),
|
| 470 |
-
init.dz || isNaN(rawPos[2])
|
| 471 |
-
? init.z
|
| 472 |
-
: rawPos[2] * (init.sz ?? 1) * (init.s ?? 1) + (init.oz ?? 0) + (init.o ?? 0),
|
| 473 |
-
)
|
| 474 |
-
: new Vector3(init.x, init.y, init.z),
|
| 475 |
-
rotation: Q_IDENTITY,
|
| 476 |
-
},
|
| 477 |
-
]
|
| 478 |
-
}
|
| 479 |
-
|
| 480 |
-
function mergeTimelines(tlsMap: Map<string, Keyframe[]>, ...tlsKey: (Keyframe[] | string)[]) {
|
| 481 |
-
const tls = tlsKey.map(resolveTimeline, tlsMap).filter(isTruely)
|
| 482 |
-
let boneName: string
|
| 483 |
-
const last = tlsKey[tlsKey.length - 1]
|
| 484 |
-
if (typeof last === 'string') boneName = last
|
| 485 |
-
else boneName = tls[tls.length - 1][0]?.boneName ?? ''
|
| 486 |
-
const results: Keyframe[] = []
|
| 487 |
-
for (const tl of tls)
|
| 488 |
-
for (const f of tl) {
|
| 489 |
-
const { frameNum } = f
|
| 490 |
-
if (frameNum < results.length && results[frameNum] != null) continue
|
| 491 |
-
const position = new Vector3()
|
| 492 |
-
const rotation = new Quaternion()
|
| 493 |
-
for (const otl of tls) {
|
| 494 |
-
if (!otl.length) continue
|
| 495 |
-
const f2 = otl[0].boneName === f.boneName ? f : lerpKeyframe(otl, frameNum)
|
| 496 |
-
position.add(tempV3.copy(f2.position).applyQuaternion(rotation))
|
| 497 |
-
rotation.multiply(f2.rotation)
|
| 498 |
-
}
|
| 499 |
-
results[frameNum] = { boneName, frameNum, position, rotation }
|
| 500 |
-
}
|
| 501 |
-
return results.filter(isTruely)
|
| 502 |
-
}
|
| 503 |
-
|
| 504 |
-
function localizeTimeline(parent: Keyframe[], child: Keyframe[]): Keyframe[]
|
| 505 |
-
function localizeTimeline(...tls: [Keyframe[], Keyframe[]]) {
|
| 506 |
-
const { boneName } = tls[1][0]
|
| 507 |
-
const results: Keyframe[] = []
|
| 508 |
-
let isChild = false
|
| 509 |
-
for (const tl of tls) {
|
| 510 |
-
for (const f of tl) {
|
| 511 |
-
const { frameNum } = f
|
| 512 |
-
if (frameNum < results.length && results[frameNum] != null) continue
|
| 513 |
-
const fp: LerpKeyframe = isChild ? lerpKeyframe(tls[0], frameNum) : f
|
| 514 |
-
const fc: LerpKeyframe = isChild ? f : lerpKeyframe(tls[1], frameNum)
|
| 515 |
-
results[frameNum] = {
|
| 516 |
-
boneName,
|
| 517 |
-
frameNum,
|
| 518 |
-
position: (fc.isNew ? fc.position : fc.position.clone()).sub(fp.position),
|
| 519 |
-
rotation: (fc.isNew ? fc.rotation : fc.rotation.clone()).multiply(
|
| 520 |
-
tempQ.copy(fp.rotation).invert(),
|
| 521 |
-
),
|
| 522 |
-
}
|
| 523 |
-
}
|
| 524 |
-
isChild = true
|
| 525 |
-
}
|
| 526 |
-
return results.filter(isTruely)
|
| 527 |
-
}
|
| 528 |
-
|
| 529 |
-
function lerpKeyframe(tl: Keyframe[], frameNum: number): LerpKeyframe {
|
| 530 |
-
if (!tl)
|
| 531 |
-
return { boneName: '', frameNum, position: V3_ZERO, rotation: Q_IDENTITY }
|
| 532 |
-
const nextIndex = tl.findIndex((keyframe) => frameNum < keyframe.frameNum)
|
| 533 |
-
switch (nextIndex) {
|
| 534 |
-
case 0:
|
| 535 |
-
return tl[0]
|
| 536 |
-
case -1:
|
| 537 |
-
return tl[tl.length - 1]
|
| 538 |
-
case frameNum:
|
| 539 |
-
return tl[frameNum]
|
| 540 |
-
}
|
| 541 |
-
const prevFrame = tl[nextIndex - 1]
|
| 542 |
-
const nextFrame = tl[nextIndex]
|
| 543 |
-
const prevFrameNum = prevFrame.frameNum
|
| 544 |
-
const nextFrameNum = nextFrame.frameNum
|
| 545 |
-
const v = (frameNum - prevFrameNum) / (nextFrameNum - prevFrameNum)
|
| 546 |
-
return {
|
| 547 |
-
boneName: tl[0].boneName,
|
| 548 |
-
frameNum,
|
| 549 |
-
position: prevFrame.position.clone().lerp(nextFrame.position, v),
|
| 550 |
-
rotation: prevFrame.rotation.clone().slerp(nextFrame.rotation, v),
|
| 551 |
-
isNew: true,
|
| 552 |
-
}
|
| 553 |
-
}
|
| 554 |
-
|
| 555 |
-
function resolveTimeline(this: Map<string, Keyframe[]>, key: Keyframe[] | string) {
|
| 556 |
-
return Array.isArray(key) ? key : this.get(key)
|
| 557 |
-
}
|
| 558 |
-
|
| 559 |
-
function sortFrames<T extends { frameNum: number }>(f: ArrayLike<T> | Iterable<T>) {
|
| 560 |
-
return (Array.isArray(f) ? (f as T[]) : Array.from(f)).sort((a, b) => a.frameNum - b.frameNum)
|
| 561 |
-
}
|
| 562 |
-
|
| 563 |
-
function getLastFrameNum(f: { frameNum: number }[]) {
|
| 564 |
-
return f.length ? f[f.length - 1].frameNum : 0
|
| 565 |
-
}
|
| 566 |
-
|
| 567 |
-
// ── VRM Offset Calculation ──────────────────────────────────────────────────
|
| 568 |
-
|
| 569 |
-
function calculatePosition(from?: Object3D | null, to?: Object3D | null) {
|
| 570 |
-
if (!from || !to) return
|
| 571 |
-
let current: Object3D | null = to
|
| 572 |
-
const chain: Object3D[] = [to]
|
| 573 |
-
while (current.parent && current !== from) {
|
| 574 |
-
chain.push(current.parent)
|
| 575 |
-
current = current.parent
|
| 576 |
-
}
|
| 577 |
-
if (current === null) return
|
| 578 |
-
chain.reverse()
|
| 579 |
-
const position = tempV3.set(0, 0, 0)
|
| 580 |
-
for (const node of chain) position.add(node.position)
|
| 581 |
-
return position.toArray()
|
| 582 |
-
}
|
| 583 |
-
|
| 584 |
-
function toOffset(vrm: VRM): VRMOffsets {
|
| 585 |
-
const { humanoid } = vrm
|
| 586 |
-
if (!humanoid) throw new Error('VRM does not have humanoid')
|
| 587 |
-
const currentPose = humanoid.getNormalizedPose()
|
| 588 |
-
humanoid.resetNormalizedPose()
|
| 589 |
-
const hips = humanoid.getNormalizedBoneNode(HumanoidBoneName.Hips)
|
| 590 |
-
const leftFoot = humanoid.getNormalizedBoneNode(HumanoidBoneName.LeftFoot)
|
| 591 |
-
const leftToe = humanoid.getNormalizedBoneNode(HumanoidBoneName.LeftToes)
|
| 592 |
-
const rightFoot = humanoid.getNormalizedBoneNode(HumanoidBoneName.RightFoot)
|
| 593 |
-
const rightToe = humanoid.getNormalizedBoneNode(HumanoidBoneName.RightToes)
|
| 594 |
-
humanoid.setNormalizedPose(currentPose)
|
| 595 |
-
return {
|
| 596 |
-
hipsOffset: calculatePosition(hips, hips),
|
| 597 |
-
leftFootOffset: calculatePosition(hips, leftFoot),
|
| 598 |
-
leftToeOffset: calculatePosition(leftFoot, leftToe),
|
| 599 |
-
rightFootOffset: calculatePosition(hips, rightFoot),
|
| 600 |
-
rightToeOffset: calculatePosition(rightFoot, rightToe),
|
| 601 |
-
}
|
| 602 |
-
}
|
| 603 |
-
|
| 604 |
-
// ── VRM Binding ─────────────────────────────────────────────────────────────
|
| 605 |
-
|
| 606 |
-
function bindToVRM(data: VMDAnimationData, vrm: VRM): AnimationClip {
|
| 607 |
-
const tracks: KeyframeTrack[] = []
|
| 608 |
-
for (const { type, name, isIK, times, values } of data.timelines) {
|
| 609 |
-
let srcName: string
|
| 610 |
-
switch (type) {
|
| 611 |
-
case 'morph': {
|
| 612 |
-
const track = vrm.expressionManager?.getExpressionTrackName(name as BlendShapePresetName)
|
| 613 |
-
if (!track) continue
|
| 614 |
-
srcName = track
|
| 615 |
-
break
|
| 616 |
-
}
|
| 617 |
-
case 'position':
|
| 618 |
-
case 'rotation': {
|
| 619 |
-
if (isIK) {
|
| 620 |
-
const handler = VRMIKHandler.get(vrm)
|
| 621 |
-
const target = handler.getAndEnableIK(name as VRMHumanBoneName)
|
| 622 |
-
if (!target) continue
|
| 623 |
-
srcName = target.name
|
| 624 |
-
} else {
|
| 625 |
-
const bone = vrm.humanoid?.getNormalizedBone(name as VRMHumanBoneName)
|
| 626 |
-
if (!bone) continue
|
| 627 |
-
srcName = bone.node.name
|
| 628 |
-
}
|
| 629 |
-
break
|
| 630 |
-
}
|
| 631 |
-
default:
|
| 632 |
-
continue
|
| 633 |
-
}
|
| 634 |
-
switch (type) {
|
| 635 |
-
case 'morph':
|
| 636 |
-
tracks.push(new NumberKeyframeTrack(srcName, times, values))
|
| 637 |
-
break
|
| 638 |
-
case 'position':
|
| 639 |
-
tracks.push(new VectorKeyframeTrack(`${srcName}.position`, times, values))
|
| 640 |
-
break
|
| 641 |
-
case 'rotation':
|
| 642 |
-
tracks.push(new QuaternionKeyframeTrack(`${srcName}.quaternion`, times, values))
|
| 643 |
-
break
|
| 644 |
-
}
|
| 645 |
-
}
|
| 646 |
-
return new AnimationClip(`clip${Date.now()}`, data.duration, tracks)
|
| 647 |
-
}
|
| 648 |
-
|
| 649 |
-
// ── Public API (matches motion-controller imports) ──────────────────────────
|
| 650 |
-
|
| 651 |
-
export async function parseVMDAnimation(url: string, vrm: VRM): Promise<VMDAnimationData> {
|
| 652 |
-
const res = await fetch(url)
|
| 653 |
-
const buffer = await res.arrayBuffer()
|
| 654 |
-
return convertVMD(buffer, toOffset(vrm))
|
| 655 |
-
}
|
| 656 |
-
|
| 657 |
-
export function bindVMDToVRM(
|
| 658 |
-
data: VMDAnimationData,
|
| 659 |
-
vrm: VRM,
|
| 660 |
-
_ikHandler: VRMIKHandler,
|
| 661 |
-
): AnimationClip {
|
| 662 |
-
return bindToVRM(data, vrm)
|
| 663 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
src/components/friend/frontend/vrm-ik-handler.ts
DELETED
|
@@ -1,236 +0,0 @@
|
|
| 1 |
-
import { VRM, VRMHumanBoneName } from '@pixiv/three-vrm'
|
| 2 |
-
import { Bone, MathUtils, Object3D, Quaternion, Vector3 } from 'three'
|
| 3 |
-
|
| 4 |
-
import { clampVector3ByRadian } from './three-helpers'
|
| 5 |
-
|
| 6 |
-
const BoneNames = VRMHumanBoneName
|
| 7 |
-
const boneNameOrder: VRMHumanBoneName[] = [
|
| 8 |
-
BoneNames.Chest,
|
| 9 |
-
BoneNames.Head,
|
| 10 |
-
BoneNames.Hips,
|
| 11 |
-
BoneNames.Jaw,
|
| 12 |
-
BoneNames.LeftEye,
|
| 13 |
-
BoneNames.LeftFoot,
|
| 14 |
-
BoneNames.LeftHand,
|
| 15 |
-
BoneNames.LeftIndexDistal,
|
| 16 |
-
BoneNames.LeftIndexIntermediate,
|
| 17 |
-
BoneNames.LeftIndexProximal,
|
| 18 |
-
BoneNames.LeftLittleDistal,
|
| 19 |
-
BoneNames.LeftLittleIntermediate,
|
| 20 |
-
BoneNames.LeftLittleProximal,
|
| 21 |
-
BoneNames.LeftLowerArm,
|
| 22 |
-
BoneNames.LeftLowerLeg,
|
| 23 |
-
BoneNames.LeftMiddleDistal,
|
| 24 |
-
BoneNames.LeftMiddleIntermediate,
|
| 25 |
-
BoneNames.LeftMiddleProximal,
|
| 26 |
-
BoneNames.LeftRingDistal,
|
| 27 |
-
BoneNames.LeftRingIntermediate,
|
| 28 |
-
BoneNames.LeftRingProximal,
|
| 29 |
-
BoneNames.LeftShoulder,
|
| 30 |
-
BoneNames.LeftThumbDistal,
|
| 31 |
-
BoneNames.LeftThumbMetacarpal,
|
| 32 |
-
BoneNames.LeftThumbProximal,
|
| 33 |
-
BoneNames.LeftToes,
|
| 34 |
-
BoneNames.LeftUpperArm,
|
| 35 |
-
BoneNames.LeftUpperLeg,
|
| 36 |
-
BoneNames.Neck,
|
| 37 |
-
BoneNames.RightEye,
|
| 38 |
-
BoneNames.RightFoot,
|
| 39 |
-
BoneNames.RightHand,
|
| 40 |
-
BoneNames.RightIndexDistal,
|
| 41 |
-
BoneNames.RightIndexIntermediate,
|
| 42 |
-
BoneNames.RightIndexProximal,
|
| 43 |
-
BoneNames.RightLittleDistal,
|
| 44 |
-
BoneNames.RightLittleIntermediate,
|
| 45 |
-
BoneNames.RightLittleProximal,
|
| 46 |
-
BoneNames.RightLowerArm,
|
| 47 |
-
BoneNames.RightLowerLeg,
|
| 48 |
-
BoneNames.RightMiddleDistal,
|
| 49 |
-
BoneNames.RightMiddleIntermediate,
|
| 50 |
-
BoneNames.RightMiddleProximal,
|
| 51 |
-
BoneNames.RightRingDistal,
|
| 52 |
-
BoneNames.RightRingIntermediate,
|
| 53 |
-
BoneNames.RightRingProximal,
|
| 54 |
-
BoneNames.RightShoulder,
|
| 55 |
-
BoneNames.RightThumbDistal,
|
| 56 |
-
BoneNames.RightThumbMetacarpal,
|
| 57 |
-
BoneNames.RightThumbProximal,
|
| 58 |
-
BoneNames.RightToes,
|
| 59 |
-
BoneNames.RightUpperArm,
|
| 60 |
-
BoneNames.RightUpperLeg,
|
| 61 |
-
BoneNames.Spine,
|
| 62 |
-
BoneNames.UpperChest,
|
| 63 |
-
]
|
| 64 |
-
const boneMap = new Map<VRMHumanBoneName, number>(boneNameOrder.map((b, i) => [b, i]))
|
| 65 |
-
|
| 66 |
-
interface IKS {
|
| 67 |
-
effector: number
|
| 68 |
-
target: number
|
| 69 |
-
iteration: number
|
| 70 |
-
links: {
|
| 71 |
-
enabled: boolean
|
| 72 |
-
index: number
|
| 73 |
-
rotationMin?: Vector3
|
| 74 |
-
rotationMax?: Vector3
|
| 75 |
-
}[]
|
| 76 |
-
minAngle?: number
|
| 77 |
-
maxAngle?: number
|
| 78 |
-
}
|
| 79 |
-
|
| 80 |
-
const quaternion = new Quaternion()
|
| 81 |
-
const targetPos = new Vector3()
|
| 82 |
-
const targetVec = new Vector3()
|
| 83 |
-
const effectorPos = new Vector3()
|
| 84 |
-
const effectorVec = new Vector3()
|
| 85 |
-
const linkPos = new Vector3()
|
| 86 |
-
const linkScale = new Vector3()
|
| 87 |
-
const axis = new Vector3()
|
| 88 |
-
|
| 89 |
-
export class VRMIKHandler {
|
| 90 |
-
private static cache = new WeakMap<VRM, VRMIKHandler>()
|
| 91 |
-
|
| 92 |
-
static get(model: VRM) {
|
| 93 |
-
let handler = this.cache.get(model)
|
| 94 |
-
if (!handler) {
|
| 95 |
-
handler = new VRMIKHandler(model)
|
| 96 |
-
this.cache.set(model, handler)
|
| 97 |
-
}
|
| 98 |
-
return handler
|
| 99 |
-
}
|
| 100 |
-
|
| 101 |
-
private targets = new Map<number, Object3D>()
|
| 102 |
-
private iks = new Map<number, IKS>()
|
| 103 |
-
private bones: Bone[]
|
| 104 |
-
private root: Object3D
|
| 105 |
-
|
| 106 |
-
private constructor(public model: VRM) {
|
| 107 |
-
const { humanoid } = this.model
|
| 108 |
-
if (!humanoid) throw new Error('VRM does not contains humanoid')
|
| 109 |
-
this.bones = boneNameOrder.map(humanoid.getRawBoneNode, humanoid) as Bone[]
|
| 110 |
-
this.root = this.bones[boneMap.get(BoneNames.Hips)!]?.parent ?? this.model.scene
|
| 111 |
-
const leftFootId = boneMap.get(BoneNames.LeftFoot)!
|
| 112 |
-
this.iks.set(leftFootId, {
|
| 113 |
-
effector: leftFootId,
|
| 114 |
-
target: leftFootId,
|
| 115 |
-
iteration: 40,
|
| 116 |
-
maxAngle: 0.5,
|
| 117 |
-
links: [
|
| 118 |
-
{
|
| 119 |
-
enabled: false,
|
| 120 |
-
index: boneMap.get(BoneNames.LeftLowerLeg)!,
|
| 121 |
-
rotationMin: new Vector3(-180, 0, 0).multiplyScalar(MathUtils.DEG2RAD),
|
| 122 |
-
rotationMax: new Vector3(0, 0, 0),
|
| 123 |
-
},
|
| 124 |
-
{
|
| 125 |
-
enabled: false,
|
| 126 |
-
index: boneMap.get(BoneNames.LeftUpperLeg)!,
|
| 127 |
-
},
|
| 128 |
-
],
|
| 129 |
-
})
|
| 130 |
-
const rightFootId = boneMap.get(BoneNames.RightFoot)!
|
| 131 |
-
this.iks.set(rightFootId, {
|
| 132 |
-
effector: rightFootId,
|
| 133 |
-
target: rightFootId,
|
| 134 |
-
iteration: 40,
|
| 135 |
-
maxAngle: 0.5,
|
| 136 |
-
links: [
|
| 137 |
-
{
|
| 138 |
-
enabled: false,
|
| 139 |
-
index: boneMap.get(BoneNames.RightLowerLeg)!,
|
| 140 |
-
rotationMin: new Vector3(-180, 0, 0).multiplyScalar(MathUtils.DEG2RAD),
|
| 141 |
-
rotationMax: new Vector3(0, 0, 0),
|
| 142 |
-
},
|
| 143 |
-
{
|
| 144 |
-
enabled: false,
|
| 145 |
-
index: boneMap.get(BoneNames.RightUpperLeg)!,
|
| 146 |
-
},
|
| 147 |
-
],
|
| 148 |
-
})
|
| 149 |
-
const leftToeId = boneMap.get(BoneNames.LeftToes)!
|
| 150 |
-
this.iks.set(leftToeId, {
|
| 151 |
-
effector: leftToeId,
|
| 152 |
-
target: leftToeId,
|
| 153 |
-
iteration: 3,
|
| 154 |
-
maxAngle: 4,
|
| 155 |
-
links: [
|
| 156 |
-
{
|
| 157 |
-
enabled: false,
|
| 158 |
-
index: leftFootId,
|
| 159 |
-
},
|
| 160 |
-
],
|
| 161 |
-
})
|
| 162 |
-
const rightToeId = boneMap.get(BoneNames.RightToes)!
|
| 163 |
-
this.iks.set(rightToeId, {
|
| 164 |
-
effector: rightToeId,
|
| 165 |
-
target: rightToeId,
|
| 166 |
-
iteration: 3,
|
| 167 |
-
maxAngle: 4,
|
| 168 |
-
links: [
|
| 169 |
-
{
|
| 170 |
-
enabled: false,
|
| 171 |
-
index: rightFootId,
|
| 172 |
-
},
|
| 173 |
-
],
|
| 174 |
-
})
|
| 175 |
-
}
|
| 176 |
-
|
| 177 |
-
getAndEnableIK(boneName: VRMHumanBoneName) {
|
| 178 |
-
return this.getTarget(boneName, true)
|
| 179 |
-
}
|
| 180 |
-
|
| 181 |
-
getTarget(boneName: VRMHumanBoneName, enable?: boolean) {
|
| 182 |
-
const boneIndex = boneMap.get(boneName)
|
| 183 |
-
if (boneIndex == null) return
|
| 184 |
-
let target = this.targets.get(boneIndex)
|
| 185 |
-
const ik = this.iks.get(boneIndex)!
|
| 186 |
-
if (enable) for (const link of ik.links) link.enabled = true
|
| 187 |
-
if (!target) {
|
| 188 |
-
target = new Object3D()
|
| 189 |
-
target.name = `${boneName}IK`
|
| 190 |
-
this.root.add(target)
|
| 191 |
-
this.targets.set(boneIndex, target)
|
| 192 |
-
}
|
| 193 |
-
return target
|
| 194 |
-
}
|
| 195 |
-
|
| 196 |
-
disableAll() {
|
| 197 |
-
for (const { links } of this.iks.values()) for (const link of links) link.enabled = false
|
| 198 |
-
}
|
| 199 |
-
|
| 200 |
-
update() {
|
| 201 |
-
for (const ik of this.iks.values()) {
|
| 202 |
-
const effector = this.bones[ik.effector]
|
| 203 |
-
const target = this.targets.get(ik.target)
|
| 204 |
-
if (!effector || !target) continue
|
| 205 |
-
targetPos.setFromMatrixPosition(target.matrixWorld)
|
| 206 |
-
const iteration = ik.iteration ?? 1
|
| 207 |
-
for (let j = 0; j < iteration; j++) {
|
| 208 |
-
let rotated = false
|
| 209 |
-
for (const { enabled, index, rotationMin, rotationMax } of ik.links) {
|
| 210 |
-
if (!enabled) break
|
| 211 |
-
const link = this.bones[index]
|
| 212 |
-
link.matrixWorld.decompose(linkPos, quaternion, linkScale)
|
| 213 |
-
quaternion.invert()
|
| 214 |
-
effectorPos.setFromMatrixPosition(effector.matrixWorld)
|
| 215 |
-
effectorVec.subVectors(effectorPos, linkPos).applyQuaternion(quaternion).normalize()
|
| 216 |
-
targetVec.subVectors(targetPos, linkPos).applyQuaternion(quaternion).normalize()
|
| 217 |
-
let angle = targetVec.dot(effectorVec)
|
| 218 |
-
if (angle > 1) angle = 1
|
| 219 |
-
else if (angle < -1) angle = -1
|
| 220 |
-
angle = Math.acos(angle)
|
| 221 |
-
if (angle < 1e-5) continue
|
| 222 |
-
if (ik.minAngle != null && angle < ik.minAngle) angle = ik.minAngle
|
| 223 |
-
if (ik.maxAngle != null && angle > ik.maxAngle) angle = ik.maxAngle
|
| 224 |
-
axis.crossVectors(effectorVec, targetVec).normalize()
|
| 225 |
-
link.quaternion.multiply(quaternion.setFromAxisAngle(axis, angle))
|
| 226 |
-
|
| 227 |
-
clampVector3ByRadian(link.rotation, rotationMin, rotationMax)
|
| 228 |
-
|
| 229 |
-
link.updateMatrixWorld(true)
|
| 230 |
-
rotated = true
|
| 231 |
-
}
|
| 232 |
-
if (!rotated) break
|
| 233 |
-
}
|
| 234 |
-
}
|
| 235 |
-
}
|
| 236 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
src/components/friend/frontend/web-file-picker.ts
DELETED
|
@@ -1,23 +0,0 @@
|
|
| 1 |
-
/**
|
| 2 |
-
* Web file picker utility — replaces Tauri's invoke('pick_*_file').
|
| 3 |
-
* Opens a hidden <input type="file"> and returns the selected File or null.
|
| 4 |
-
*/
|
| 5 |
-
export function pickFile(accept: string): Promise<File | null> {
|
| 6 |
-
return new Promise((resolve) => {
|
| 7 |
-
const input = document.createElement('input')
|
| 8 |
-
input.type = 'file'
|
| 9 |
-
input.accept = accept
|
| 10 |
-
input.style.display = 'none'
|
| 11 |
-
input.addEventListener('change', () => {
|
| 12 |
-
const file = input.files?.[0] ?? null
|
| 13 |
-
resolve(file)
|
| 14 |
-
document.body.removeChild(input)
|
| 15 |
-
})
|
| 16 |
-
input.addEventListener('cancel', () => {
|
| 17 |
-
resolve(null)
|
| 18 |
-
document.body.removeChild(input)
|
| 19 |
-
})
|
| 20 |
-
document.body.appendChild(input)
|
| 21 |
-
input.click()
|
| 22 |
-
})
|
| 23 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
src/friend/FriendService.ts
CHANGED
|
@@ -20,6 +20,7 @@ import { stripForTts } from './text-utils.js';
|
|
| 20 |
import { edgeTts, qwenTts, registerAudioFile, getAudioFile } from './tts.js';
|
| 21 |
import { splitSentences } from './text-utils.js';
|
| 22 |
import { SileroVad } from './voice/vad-service.js';
|
|
|
|
| 23 |
import { readFileSync } from 'node:fs';
|
| 24 |
import { buildVrmSystemPrompt } from '../skills/bundled/friendPrompt.js';
|
| 25 |
|
|
@@ -79,6 +80,8 @@ class FriendService {
|
|
| 79 |
private _flushing = false;
|
| 80 |
/** Silero VAD instance (real ML-based voice activity detection) */
|
| 81 |
private vadInstance: SileroVad | null = null;
|
|
|
|
|
|
|
| 82 |
/** When muted, audio from arecord is not forwarded to STT or VAD (prevents echo) */
|
| 83 |
private muted = false;
|
| 84 |
/** Timer to automatically unmute after estimated TTS playback */
|
|
@@ -118,13 +121,13 @@ class FriendService {
|
|
| 118 |
);
|
| 119 |
},
|
| 120 |
}, {
|
| 121 |
-
//
|
| 122 |
-
positiveSpeechThreshold: 0.
|
| 123 |
-
negativeSpeechThreshold: 0.
|
| 124 |
-
preSpeechTriggerFrames:
|
| 125 |
-
minSpeechFrames:
|
| 126 |
-
redemptionFrames:
|
| 127 |
-
rmsThreshold: 0.
|
| 128 |
});
|
| 129 |
vad.init().then(() => {
|
| 130 |
this.vadInstance = vad;
|
|
@@ -265,6 +268,7 @@ class FriendService {
|
|
| 265 |
// audio chunks flow to VAD via the arecord data callback (see loadAudioCapture)
|
| 266 |
if (this.vadInstance) {
|
| 267 |
try {
|
|
|
|
| 268 |
this.vadInstance.start();
|
| 269 |
} catch (e) {
|
| 270 |
console.warn('[FriendService] VAD start error (non-fatal):', e);
|
|
@@ -803,6 +807,8 @@ class FriendService {
|
|
| 803 |
for (let i = 0; i < float32.length; i++) {
|
| 804 |
float32[i] = c.readInt16LE(i * 2) / 32768;
|
| 805 |
}
|
|
|
|
|
|
|
| 806 |
this.vadInstance.processAudio(float32).catch(() => {});
|
| 807 |
}
|
| 808 |
};
|
|
|
|
| 20 |
import { edgeTts, qwenTts, registerAudioFile, getAudioFile } from './tts.js';
|
| 21 |
import { splitSentences } from './text-utils.js';
|
| 22 |
import { SileroVad } from './voice/vad-service.js';
|
| 23 |
+
import { HighPassFilter } from './voice/highpass-filter.js';
|
| 24 |
import { readFileSync } from 'node:fs';
|
| 25 |
import { buildVrmSystemPrompt } from '../skills/bundled/friendPrompt.js';
|
| 26 |
|
|
|
|
| 80 |
private _flushing = false;
|
| 81 |
/** Silero VAD instance (real ML-based voice activity detection) */
|
| 82 |
private vadInstance: SileroVad | null = null;
|
| 83 |
+
/** High-pass filter to remove low-frequency noise (fan/AC hum) before VAD */
|
| 84 |
+
private highpassFilter = new HighPassFilter(80);
|
| 85 |
/** When muted, audio from arecord is not forwarded to STT or VAD (prevents echo) */
|
| 86 |
private muted = false;
|
| 87 |
/** Timer to automatically unmute after estimated TTS playback */
|
|
|
|
| 121 |
);
|
| 122 |
},
|
| 123 |
}, {
|
| 124 |
+
// Strict thresholds for near-field mic — rejects keyboard/ambient noise
|
| 125 |
+
positiveSpeechThreshold: 0.90, // need 90% confidence (was 80%)
|
| 126 |
+
negativeSpeechThreshold: 0.40, // drop below 40% to end (was 50%)
|
| 127 |
+
preSpeechTriggerFrames: 16, // require ~512ms sustained speech (was 480ms)
|
| 128 |
+
minSpeechFrames: 8, // ~256ms min speech (was 192ms)
|
| 129 |
+
redemptionFrames: 15, // ~480ms silence before segment end (was 640ms)
|
| 130 |
+
rmsThreshold: 0.015, // -36dBFS noise floor (was -40dBFS)
|
| 131 |
});
|
| 132 |
vad.init().then(() => {
|
| 133 |
this.vadInstance = vad;
|
|
|
|
| 268 |
// audio chunks flow to VAD via the arecord data callback (see loadAudioCapture)
|
| 269 |
if (this.vadInstance) {
|
| 270 |
try {
|
| 271 |
+
this.highpassFilter.reset(); // reset filter state for new session
|
| 272 |
this.vadInstance.start();
|
| 273 |
} catch (e) {
|
| 274 |
console.warn('[FriendService] VAD start error (non-fatal):', e);
|
|
|
|
| 807 |
for (let i = 0; i < float32.length; i++) {
|
| 808 |
float32[i] = c.readInt16LE(i * 2) / 32768;
|
| 809 |
}
|
| 810 |
+
// High-pass filter: remove sub-80Hz noise (fan/AC hum) before VAD
|
| 811 |
+
this.highpassFilter.process(float32);
|
| 812 |
this.vadInstance.processAudio(float32).catch(() => {});
|
| 813 |
}
|
| 814 |
};
|
src/friend/prefs.ts
CHANGED
|
@@ -17,11 +17,7 @@ export interface FriendPrefs {
|
|
| 17 |
tracking?: 'mouse' | 'camera';
|
| 18 |
volume?: number;
|
| 19 |
uiAlign?: 'left' | 'right';
|
| 20 |
-
screenObserve?: boolean;
|
| 21 |
-
screenObserveInterval?: number;
|
| 22 |
language?: 'zh' | 'en';
|
| 23 |
-
currentDance?: string;
|
| 24 |
-
hideMood?: boolean;
|
| 25 |
/** STT provider: browser | anthropic | local | doubao | groq */
|
| 26 |
sttProvider?: 'browser' | 'anthropic' | 'local' | 'doubao' | 'groq';
|
| 27 |
/** STT language override (e.g. 'en', 'zh', 'ja') */
|
|
|
|
| 17 |
tracking?: 'mouse' | 'camera';
|
| 18 |
volume?: number;
|
| 19 |
uiAlign?: 'left' | 'right';
|
|
|
|
|
|
|
| 20 |
language?: 'zh' | 'en';
|
|
|
|
|
|
|
| 21 |
/** STT provider: browser | anthropic | local | doubao | groq */
|
| 22 |
sttProvider?: 'browser' | 'anthropic' | 'local' | 'doubao' | 'groq';
|
| 23 |
/** STT language override (e.g. 'en', 'zh', 'ja') */
|
src/friend/sse.ts
CHANGED
|
@@ -38,6 +38,7 @@ export type VrmBroadcastPayload = {
|
|
| 38 |
text?: string;
|
| 39 |
emotion?: string;
|
| 40 |
emotionIntensity?: number;
|
|
|
|
| 41 |
audioUrl?: string;
|
| 42 |
audioIndex?: number;
|
| 43 |
clearText?: boolean;
|
|
|
|
| 38 |
text?: string;
|
| 39 |
emotion?: string;
|
| 40 |
emotionIntensity?: number;
|
| 41 |
+
action?: string;
|
| 42 |
audioUrl?: string;
|
| 43 |
audioIndex?: number;
|
| 44 |
clearText?: boolean;
|
src/friend/voice/highpass-filter.ts
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/**
|
| 2 |
+
* First-order IIR high-pass filter for DC offset removal and low-frequency noise rejection.
|
| 3 |
+
*
|
| 4 |
+
* Removes subsonic rumble, fan hum (50/60Hz), and AC line noise before VAD
|
| 5 |
+
* processing — these are not human voice and can confuse the VAD model.
|
| 6 |
+
*
|
| 7 |
+
* Tuned for 16kHz sample rate with adjustable cutoff (~80Hz default).
|
| 8 |
+
* Preserves all human voice frequencies (fundamental 85-255Hz + formants).
|
| 9 |
+
*
|
| 10 |
+
* Formula: y[n] = x[n] - x[n-1] + α ⋅ y[n-1]
|
| 11 |
+
* where α = exp(-2π ⋅ fc / fs)
|
| 12 |
+
*/
|
| 13 |
+
|
| 14 |
+
export class HighPassFilter {
|
| 15 |
+
private prevX = 0
|
| 16 |
+
private prevY = 0
|
| 17 |
+
private readonly alpha: number
|
| 18 |
+
|
| 19 |
+
/**
|
| 20 |
+
* @param cutoffHz -3dB cutoff frequency in Hz (default: 80 — below male voice fundamental)
|
| 21 |
+
* @param sampleRate input sample rate (default: 16000 — Silero VAD sample rate)
|
| 22 |
+
*/
|
| 23 |
+
constructor(cutoffHz = 80, sampleRate = 16000) {
|
| 24 |
+
const omega = 2 * Math.PI * cutoffHz / sampleRate
|
| 25 |
+
this.alpha = Math.exp(-omega)
|
| 26 |
+
}
|
| 27 |
+
|
| 28 |
+
/**
|
| 29 |
+
* Process a Float32Array of audio samples in-place.
|
| 30 |
+
* Zero-copy — mutates the input array.
|
| 31 |
+
*/
|
| 32 |
+
process(samples: Float32Array): void {
|
| 33 |
+
const a = this.alpha
|
| 34 |
+
let x1 = this.prevX
|
| 35 |
+
let y1 = this.prevY
|
| 36 |
+
|
| 37 |
+
for (let i = 0; i < samples.length; i++) {
|
| 38 |
+
const x0 = samples[i]
|
| 39 |
+
const y0 = x0 - x1 + a * y1
|
| 40 |
+
samples[i] = y0
|
| 41 |
+
x1 = x0
|
| 42 |
+
y1 = y0
|
| 43 |
+
}
|
| 44 |
+
|
| 45 |
+
this.prevX = x1
|
| 46 |
+
this.prevY = y1
|
| 47 |
+
}
|
| 48 |
+
|
| 49 |
+
/** Reset filter state. Call when starting a new capture session. */
|
| 50 |
+
reset(): void {
|
| 51 |
+
this.prevX = 0
|
| 52 |
+
this.prevY = 0
|
| 53 |
+
}
|
| 54 |
+
}
|
src/server/api/friend.ts
CHANGED
|
@@ -1,31 +1,16 @@
|
|
| 1 |
/**
|
| 2 |
* Friend API — VRM avatar frontend HTTP routes for VersperClaw.
|
| 3 |
*
|
| 4 |
-
*
|
| 5 |
-
* communicates with. Replaces the OpenClaw gateway routing.
|
| 6 |
*/
|
| 7 |
import path from 'node:path';
|
| 8 |
-
import { existsSync, readFileSync
|
| 9 |
import {
|
| 10 |
createSseResponse,
|
| 11 |
-
broadcastToVrm,
|
| 12 |
} from '../../friend/sse.js';
|
| 13 |
import { getPrefs, setPrefs, updatePrefs, type FriendPrefs } from '../../friend/prefs.js';
|
| 14 |
import { edgeTts, qwenTts, registerAudioFile, getAudioFile } from '../../friend/tts.js';
|
| 15 |
-
import { stripForTts } from '../../friend/text-utils.js';
|
| 16 |
import { friendService } from '../../friend/FriendService.js';
|
| 17 |
-
import { transcribeAudioFile } from '../../friend/stt-service.js';
|
| 18 |
-
|
| 19 |
-
const GATEWAY_URL = `http://127.0.0.1:3456`;
|
| 20 |
-
|
| 21 |
-
/** Resolved at startup from the Bun.serve config. Updated by the server. */
|
| 22 |
-
let serverHost = '127.0.0.1';
|
| 23 |
-
let serverPort = 3456;
|
| 24 |
-
|
| 25 |
-
export function setFriendServerInfo(host: string, port: number): void {
|
| 26 |
-
serverHost = host;
|
| 27 |
-
serverPort = port;
|
| 28 |
-
}
|
| 29 |
|
| 30 |
const MIME_TYPES: Record<string, string> = {
|
| 31 |
'.mp3': 'audio/mpeg',
|
|
@@ -35,29 +20,6 @@ const MIME_TYPES: Record<string, string> = {
|
|
| 35 |
'.webm': 'audio/webm',
|
| 36 |
};
|
| 37 |
|
| 38 |
-
const IMAGE_MIME: Record<string, string> = {
|
| 39 |
-
'.png': 'image/png', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg',
|
| 40 |
-
'.gif': 'image/gif', '.webp': 'image/webp', '.svg': 'image/svg+xml',
|
| 41 |
-
'.bmp': 'image/bmp',
|
| 42 |
-
};
|
| 43 |
-
|
| 44 |
-
// Workspace paths
|
| 45 |
-
const homeDir = process.env.HOME || process.env.USERPROFILE || '';
|
| 46 |
-
// Session tracking for dance import (MP3 follows VMD in user flow)
|
| 47 |
-
let lastImportedDanceId: string | null = null;
|
| 48 |
-
const workspaceRoot = path.join(homeDir, '.config', 'VersperClaw', 'friend');
|
| 49 |
-
|
| 50 |
-
// Persona files
|
| 51 |
-
const identityPath = path.join(workspaceRoot, 'IDENTITY.md');
|
| 52 |
-
const soulPath = path.join(workspaceRoot, 'SOUL.md');
|
| 53 |
-
|
| 54 |
-
// Models & dances
|
| 55 |
-
const customModelsDir = path.join(workspaceRoot, 'models');
|
| 56 |
-
const customDancesDir = path.join(workspaceRoot, 'dances');
|
| 57 |
-
|
| 58 |
-
// Media file registry
|
| 59 |
-
const mediaFileRegistry = new Map<string, string>();
|
| 60 |
-
|
| 61 |
function corsHeaders(): Record<string, string> {
|
| 62 |
return {
|
| 63 |
'Access-Control-Allow-Origin': '*',
|
|
@@ -74,10 +36,6 @@ function notFound(msg = 'Not found'): Response {
|
|
| 74 |
return jsonResponse({ error: msg }, 404);
|
| 75 |
}
|
| 76 |
|
| 77 |
-
function readJsonBody(req: Request): Promise<unknown> {
|
| 78 |
-
return req.json();
|
| 79 |
-
}
|
| 80 |
-
|
| 81 |
/**
|
| 82 |
* Route handler for /plugins/friend/*
|
| 83 |
*/
|
|
@@ -90,12 +48,12 @@ export async function handleFriendApi(req: Request, url: URL): Promise<Response>
|
|
| 90 |
return new Response(null, { status: 204, headers: corsHeaders() });
|
| 91 |
}
|
| 92 |
|
| 93 |
-
// ── SSE Events endpoint
|
| 94 |
if (pathname === '/plugins/friend/events' && method === 'GET') {
|
| 95 |
return createSseResponse();
|
| 96 |
}
|
| 97 |
|
| 98 |
-
// ── Audio serving
|
| 99 |
if (pathname.startsWith('/plugins/friend/audio/') && method === 'GET') {
|
| 100 |
const audioId = pathname.split('/plugins/friend/audio/')[1]?.split('?')[0];
|
| 101 |
if (!audioId) return jsonResponse({ error: 'missing audio id' }, 400);
|
|
@@ -111,72 +69,7 @@ export async function handleFriendApi(req: Request, url: URL): Promise<Response>
|
|
| 111 |
});
|
| 112 |
}
|
| 113 |
|
| 114 |
-
// ──
|
| 115 |
-
if (pathname.startsWith('/plugins/friend/media/') && method === 'GET') {
|
| 116 |
-
const mediaId = pathname.split('/plugins/friend/media/')[1]?.split('?')[0];
|
| 117 |
-
if (!mediaId) return jsonResponse({ error: 'missing media id' }, 400);
|
| 118 |
-
|
| 119 |
-
const filePath = mediaFileRegistry.get(mediaId);
|
| 120 |
-
if (!filePath || !existsSync(filePath)) return notFound('media not found');
|
| 121 |
-
|
| 122 |
-
const ext = path.extname(filePath).toLowerCase();
|
| 123 |
-
const contentType = IMAGE_MIME[ext] ?? 'application/octet-stream';
|
| 124 |
-
const data = readFileSync(filePath);
|
| 125 |
-
return new Response(data, {
|
| 126 |
-
headers: { 'Content-Type': contentType, 'Cache-Control': 'public, max-age=300', ...corsHeaders() },
|
| 127 |
-
});
|
| 128 |
-
}
|
| 129 |
-
|
| 130 |
-
// ── Chat endpoint (POST /plugins/friend/chat) ──
|
| 131 |
-
if (pathname === '/plugins/friend/chat' && method === 'POST') {
|
| 132 |
-
const body = await readJsonBody(req) as any;
|
| 133 |
-
const message = body?.message;
|
| 134 |
-
if (!message) return jsonResponse({ error: 'message required' }, 400);
|
| 135 |
-
|
| 136 |
-
// Enqueue the message into the main CLI conversation via FriendService.
|
| 137 |
-
// The AI response is tracked by useFriendBridge in the REPL and
|
| 138 |
-
// broadcast back to the VRM display via SSE automatically.
|
| 139 |
-
try {
|
| 140 |
-
await friendService.start();
|
| 141 |
-
friendService.sendText(message);
|
| 142 |
-
} catch (err) {
|
| 143 |
-
console.error('[Friend] chat dispatch error:', err);
|
| 144 |
-
broadcastToVrm({
|
| 145 |
-
text: `Connection error: ${err instanceof Error ? err.message : String(err)}`,
|
| 146 |
-
});
|
| 147 |
-
broadcastToVrm({ replyDone: true });
|
| 148 |
-
}
|
| 149 |
-
|
| 150 |
-
return jsonResponse({ ok: true });
|
| 151 |
-
}
|
| 152 |
-
|
| 153 |
-
// ── STT segment from browser VAD (POST /plugins/friend/voice/stt-segment) ──
|
| 154 |
-
if (pathname === '/plugins/friend/voice/stt-segment' && method === 'POST') {
|
| 155 |
-
try {
|
| 156 |
-
const contentType = req.headers.get('content-type') || '';
|
| 157 |
-
let audioBuffer: Buffer;
|
| 158 |
-
|
| 159 |
-
if (contentType.includes('multipart/form-data') || contentType.includes('application/octet-stream')) {
|
| 160 |
-
const blob = await req.blob();
|
| 161 |
-
audioBuffer = Buffer.from(await blob.arrayBuffer());
|
| 162 |
-
} else {
|
| 163 |
-
// Accept raw PCM or WAV as binary body
|
| 164 |
-
audioBuffer = Buffer.from(await req.arrayBuffer());
|
| 165 |
-
}
|
| 166 |
-
|
| 167 |
-
if (audioBuffer.length < 100) {
|
| 168 |
-
return jsonResponse({ error: 'audio too short' }, 400);
|
| 169 |
-
}
|
| 170 |
-
|
| 171 |
-
const transcript = await friendService.transcribeAudioSegment(audioBuffer);
|
| 172 |
-
return jsonResponse({ ok: true, text: transcript });
|
| 173 |
-
} catch (err) {
|
| 174 |
-
console.error('[Friend] STT segment error:', err);
|
| 175 |
-
return jsonResponse({ error: String(err) }, 500);
|
| 176 |
-
}
|
| 177 |
-
}
|
| 178 |
-
|
| 179 |
-
// ── Voice capture (POST /plugins/friend/voice/start, /start-vad, /stop, /status) ──
|
| 180 |
if (pathname === '/plugins/friend/voice/start' && method === 'POST') {
|
| 181 |
try {
|
| 182 |
await friendService.startVoiceCapture();
|
|
@@ -199,21 +92,7 @@ export async function handleFriendApi(req: Request, url: URL): Promise<Response>
|
|
| 199 |
return jsonResponse(friendService.getCaptureStatus());
|
| 200 |
}
|
| 201 |
|
| 202 |
-
// ──
|
| 203 |
-
if (pathname === '/plugins/friend/touch' && method === 'POST') {
|
| 204 |
-
const body = await readJsonBody(req) as any;
|
| 205 |
-
const region = body?.region;
|
| 206 |
-
const prompt = body?.prompt;
|
| 207 |
-
if (!region || !prompt) return jsonResponse({ error: 'region and prompt required' }, 400);
|
| 208 |
-
|
| 209 |
-
setImmediate(() => {
|
| 210 |
-
broadcastToVrm({ text: `[Touch: ${region}] ${prompt}` });
|
| 211 |
-
});
|
| 212 |
-
|
| 213 |
-
return jsonResponse({ ok: true });
|
| 214 |
-
}
|
| 215 |
-
|
| 216 |
-
// ── Voice settings (GET/POST /plugins/friend/voice) ──
|
| 217 |
if (pathname === '/plugins/friend/voice') {
|
| 218 |
if (method === 'GET') {
|
| 219 |
const prefs = getPrefs();
|
|
@@ -224,9 +103,8 @@ export async function handleFriendApi(req: Request, url: URL): Promise<Response>
|
|
| 224 |
qwenModel: prefs.qwenModel ?? 'qwen3-tts-flash',
|
| 225 |
});
|
| 226 |
}
|
| 227 |
-
|
| 228 |
if (method === 'POST') {
|
| 229 |
-
const body = await
|
| 230 |
const patch: Partial<FriendPrefs> = {};
|
| 231 |
if (body.voice !== undefined) patch.voice = body.voice || undefined;
|
| 232 |
if (body.provider !== undefined) patch.provider = body.provider || undefined;
|
|
@@ -235,11 +113,10 @@ export async function handleFriendApi(req: Request, url: URL): Promise<Response>
|
|
| 235 |
setPrefs(updatePrefs(patch));
|
| 236 |
return jsonResponse({ ok: true });
|
| 237 |
}
|
| 238 |
-
|
| 239 |
return new Response(null, { status: 405 });
|
| 240 |
}
|
| 241 |
|
| 242 |
-
// ── STT config
|
| 243 |
if (pathname === '/plugins/friend/stt/config' && method === 'GET') {
|
| 244 |
const prefs = getPrefs();
|
| 245 |
return jsonResponse({
|
|
@@ -248,28 +125,33 @@ export async function handleFriendApi(req: Request, url: URL): Promise<Response>
|
|
| 248 |
});
|
| 249 |
}
|
| 250 |
|
| 251 |
-
// ── STT
|
| 252 |
-
if (pathname === '/plugins/friend/
|
| 253 |
try {
|
| 254 |
-
|
| 255 |
-
const
|
| 256 |
-
if (
|
| 257 |
-
|
| 258 |
-
|
| 259 |
-
|
| 260 |
-
|
| 261 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 262 |
} catch (err) {
|
| 263 |
return jsonResponse({ error: String(err) }, 500);
|
| 264 |
}
|
| 265 |
}
|
| 266 |
|
| 267 |
-
// ── TTS Preview
|
| 268 |
if (pathname === '/plugins/friend/preview' && method === 'POST') {
|
| 269 |
-
const body = await
|
| 270 |
const voice = body.voice as string | undefined;
|
| 271 |
const provider = body.provider as string | undefined;
|
| 272 |
-
const text = '
|
| 273 |
|
| 274 |
try {
|
| 275 |
const prefs = getPrefs();
|
|
@@ -285,10 +167,9 @@ export async function handleFriendApi(req: Request, url: URL): Promise<Response>
|
|
| 285 |
} else {
|
| 286 |
result = await edgeTts({ text, voice: voice || prefs.voice });
|
| 287 |
}
|
| 288 |
-
|
| 289 |
if (result.success && result.audioPath) {
|
| 290 |
const audioId = registerAudioFile(result.audioPath);
|
| 291 |
-
return jsonResponse({ audioUrl: `
|
| 292 |
}
|
| 293 |
return jsonResponse({ error: result.error || 'TTS failed' });
|
| 294 |
} catch (err) {
|
|
@@ -296,7 +177,7 @@ export async function handleFriendApi(req: Request, url: URL): Promise<Response>
|
|
| 296 |
}
|
| 297 |
}
|
| 298 |
|
| 299 |
-
// ── General settings (
|
| 300 |
if (pathname === '/plugins/friend/settings') {
|
| 301 |
if (method === 'GET') {
|
| 302 |
const prefs = getPrefs();
|
|
@@ -308,18 +189,13 @@ export async function handleFriendApi(req: Request, url: URL): Promise<Response>
|
|
| 308 |
tracking: prefs.tracking,
|
| 309 |
volume: prefs.volume,
|
| 310 |
uiAlign: prefs.uiAlign,
|
| 311 |
-
screenObserve: prefs.screenObserve,
|
| 312 |
-
screenObserveInterval: prefs.screenObserveInterval,
|
| 313 |
-
currentDance: prefs.currentDance,
|
| 314 |
-
language: prefs.language,
|
| 315 |
-
hideMood: prefs.hideMood,
|
| 316 |
sttProvider: prefs.sttProvider || 'browser',
|
| 317 |
sttLanguage: prefs.sttLanguage || 'zh',
|
|
|
|
| 318 |
});
|
| 319 |
}
|
| 320 |
-
|
| 321 |
if (method === 'POST') {
|
| 322 |
-
const body = await
|
| 323 |
const patch: Partial<FriendPrefs> = {};
|
| 324 |
if (body.modelPath !== undefined) patch.modelPath = body.modelPath;
|
| 325 |
if (body.ttsEnabled !== undefined) patch.ttsEnabled = body.ttsEnabled;
|
|
@@ -328,22 +204,24 @@ export async function handleFriendApi(req: Request, url: URL): Promise<Response>
|
|
| 328 |
if (body.tracking !== undefined) patch.tracking = body.tracking;
|
| 329 |
if (body.volume !== undefined) patch.volume = body.volume;
|
| 330 |
if (body.uiAlign !== undefined) patch.uiAlign = body.uiAlign;
|
| 331 |
-
if (body.screenObserve !== undefined) patch.screenObserve = body.screenObserve;
|
| 332 |
-
if (body.screenObserveInterval !== undefined) patch.screenObserveInterval = body.screenObserveInterval;
|
| 333 |
-
if (body.currentDance !== undefined) patch.currentDance = body.currentDance;
|
| 334 |
-
if (body.language !== undefined) patch.language = body.language;
|
| 335 |
-
if (body.hideMood !== undefined) patch.hideMood = body.hideMood;
|
| 336 |
if (body.sttProvider !== undefined) patch.sttProvider = body.sttProvider;
|
| 337 |
if (body.sttLanguage !== undefined) patch.sttLanguage = body.sttLanguage;
|
|
|
|
| 338 |
if (body.groqApiKey !== undefined) patch.groqApiKey = body.groqApiKey;
|
| 339 |
setPrefs(updatePrefs(patch));
|
| 340 |
return jsonResponse({ ok: true });
|
| 341 |
}
|
| 342 |
-
|
| 343 |
return new Response(null, { status: 405 });
|
| 344 |
}
|
| 345 |
|
| 346 |
-
// ── Persona (
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 347 |
if (pathname === '/plugins/friend/persona') {
|
| 348 |
if (method === 'GET') {
|
| 349 |
let soul = '';
|
|
@@ -352,9 +230,9 @@ export async function handleFriendApi(req: Request, url: URL): Promise<Response>
|
|
| 352 |
try { if (existsSync(identityPath)) identity = readFileSync(identityPath, 'utf8'); } catch { /* */ }
|
| 353 |
return jsonResponse({ soul, identity, soulPath, identityPath });
|
| 354 |
}
|
| 355 |
-
|
| 356 |
if (method === 'POST') {
|
| 357 |
-
const body = await
|
|
|
|
| 358 |
if (body.soul !== undefined) {
|
| 359 |
mkdirSync(path.dirname(soulPath), { recursive: true });
|
| 360 |
writeFileSync(soulPath, body.soul, 'utf8');
|
|
@@ -365,428 +243,28 @@ export async function handleFriendApi(req: Request, url: URL): Promise<Response>
|
|
| 365 |
}
|
| 366 |
return jsonResponse({ ok: true });
|
| 367 |
}
|
| 368 |
-
|
| 369 |
return new Response(null, { status: 405 });
|
| 370 |
}
|
| 371 |
|
| 372 |
-
// ──
|
| 373 |
-
if (pathname === '/plugins/friend/
|
| 374 |
-
|
| 375 |
-
let custom: string[] = [];
|
| 376 |
-
if (existsSync(customModelsDir)) {
|
| 377 |
-
custom = readdirSync(customModelsDir)
|
| 378 |
-
.filter((f: string) => f.toLowerCase().endsWith('.vrm'))
|
| 379 |
-
.map((f: string) => `${GATEWAY_URL}/plugins/friend/model/serve/${f}`);
|
| 380 |
-
}
|
| 381 |
-
return jsonResponse({ models: custom });
|
| 382 |
-
} catch {
|
| 383 |
-
return jsonResponse({ models: [] });
|
| 384 |
-
}
|
| 385 |
-
}
|
| 386 |
-
|
| 387 |
-
// ── Model serve (GET /plugins/friend/model/serve/:file) ──
|
| 388 |
-
if (pathname.startsWith('/plugins/friend/model/serve/') && method === 'GET') {
|
| 389 |
-
const fileName = decodeURIComponent(pathname.split('/plugins/friend/model/serve/')[1]?.split('?')[0] ?? '');
|
| 390 |
-
if (!fileName || fileName.includes('..') || fileName.includes('/')) {
|
| 391 |
-
return jsonResponse({ error: 'invalid file name' }, 400);
|
| 392 |
-
}
|
| 393 |
-
const filePath = path.join(customModelsDir, fileName);
|
| 394 |
-
if (!existsSync(filePath)) return notFound('model not found');
|
| 395 |
-
|
| 396 |
-
const data = readFileSync(filePath);
|
| 397 |
-
return new Response(data, {
|
| 398 |
-
headers: {
|
| 399 |
-
'Content-Type': 'application/octet-stream',
|
| 400 |
-
'Content-Length': String(data.length),
|
| 401 |
-
'Cache-Control': 'public, max-age=3600',
|
| 402 |
-
...corsHeaders(),
|
| 403 |
-
},
|
| 404 |
-
});
|
| 405 |
-
}
|
| 406 |
-
|
| 407 |
-
// ── History (GET /plugins/friend/history) ──
|
| 408 |
-
if (pathname === '/plugins/friend/history' && method === 'GET') {
|
| 409 |
-
try {
|
| 410 |
-
const stateDir = path.join(homeDir, '.local', 'share', 'VersperClaw');
|
| 411 |
-
const sessionsDir = path.join(stateDir, 'sessions');
|
| 412 |
-
|
| 413 |
-
let messages: any[] = [];
|
| 414 |
-
if (existsSync(sessionsDir)) {
|
| 415 |
-
const files = readdirSync(sessionsDir)
|
| 416 |
-
.filter((f: string) => f.endsWith('.jsonl'))
|
| 417 |
-
.map((f: string) => ({
|
| 418 |
-
name: f,
|
| 419 |
-
mtime: statSync(path.join(sessionsDir, f)).mtimeMs,
|
| 420 |
-
}))
|
| 421 |
-
.sort((a: any, b: any) => b.mtime - a.mtime);
|
| 422 |
-
|
| 423 |
-
if (files.length > 0) {
|
| 424 |
-
const jsonlPath = path.join(sessionsDir, files[0].name);
|
| 425 |
-
const lines = readFileSync(jsonlPath, 'utf8').split('\n').filter((l: string) => l.trim());
|
| 426 |
-
for (const line of lines) {
|
| 427 |
-
try {
|
| 428 |
-
const obj = JSON.parse(line);
|
| 429 |
-
if (obj.message) messages.push(obj.message);
|
| 430 |
-
} catch { /* skip malformed */ }
|
| 431 |
-
}
|
| 432 |
-
}
|
| 433 |
-
}
|
| 434 |
-
|
| 435 |
-
let agentName = '';
|
| 436 |
-
try {
|
| 437 |
-
if (existsSync(identityPath)) {
|
| 438 |
-
const identity = readFileSync(identityPath, 'utf8');
|
| 439 |
-
const nameMatch = identity.match(/\*\*Name:\*\*\s*(.+)/i) || identity.match(/^#\s+(.+)/m);
|
| 440 |
-
if (nameMatch) agentName = nameMatch[1].trim();
|
| 441 |
-
}
|
| 442 |
-
} catch { /* */ }
|
| 443 |
-
|
| 444 |
-
const formatted = messages
|
| 445 |
-
.map((msg: any) => ({
|
| 446 |
-
role: msg.role === 'model' ? 'assistant' : msg.role,
|
| 447 |
-
content: typeof msg.content === 'string' ? msg.content : '',
|
| 448 |
-
timestamp: msg.timestamp,
|
| 449 |
-
}))
|
| 450 |
-
.filter((m: any) => (m.role === 'user' || m.role === 'assistant') && m.content)
|
| 451 |
-
.slice(-100);
|
| 452 |
-
|
| 453 |
-
return jsonResponse({ messages: formatted, agentName: agentName || undefined });
|
| 454 |
-
} catch (err) {
|
| 455 |
-
return jsonResponse({ error: String(err) }, 500);
|
| 456 |
-
}
|
| 457 |
-
}
|
| 458 |
-
|
| 459 |
-
// ── Clear context (POST /plugins/friend/context/clear) ──
|
| 460 |
-
if (pathname === '/plugins/friend/context/clear' && method === 'POST') {
|
| 461 |
-
broadcastToVrm({ clearText: true });
|
| 462 |
-
friendService.sendText('/new');
|
| 463 |
return jsonResponse({ ok: true });
|
| 464 |
}
|
| 465 |
|
| 466 |
-
// ──
|
| 467 |
-
if (pathname === '/plugins/friend/
|
| 468 |
-
const body = await
|
| 469 |
-
const
|
| 470 |
-
if (
|
| 471 |
-
|
| 472 |
-
const prefs = getPrefs();
|
| 473 |
-
const currentMood = (prefs as any)._moodIndex ?? 60;
|
| 474 |
-
const cap = body.max != null ? Math.round(Number(body.max)) : 100;
|
| 475 |
-
const newMood = Math.max(0, Math.min(cap, currentMood + delta));
|
| 476 |
-
(prefs as any)._moodIndex = newMood;
|
| 477 |
-
setPrefs(prefs);
|
| 478 |
-
|
| 479 |
-
broadcastToVrm({ moodDelta: delta, moodIndex: newMood });
|
| 480 |
-
return jsonResponse({ ok: true, moodIndex: newMood });
|
| 481 |
-
}
|
| 482 |
-
|
| 483 |
-
// ── Session memo (POST /plugins/friend/session/memo) ──
|
| 484 |
-
if (pathname === '/plugins/friend/session/memo' && method === 'POST') {
|
| 485 |
-
const body = await readJsonBody(req) as any;
|
| 486 |
-
const text = typeof body.text === 'string' ? body.text.trim() : '';
|
| 487 |
-
if (!text) return jsonResponse({ error: 'text required' }, 400);
|
| 488 |
-
|
| 489 |
-
try {
|
| 490 |
-
const stateDir = path.join(homeDir, '.local', 'share', 'VersperClaw');
|
| 491 |
-
const sessionLog = path.join(stateDir, 'friend-memo.jsonl');
|
| 492 |
-
mkdirSync(path.dirname(sessionLog), { recursive: true });
|
| 493 |
-
appendFileSync(sessionLog, JSON.stringify({ role: 'user', content: text }) + '\n', 'utf8');
|
| 494 |
-
return jsonResponse({ ok: true, written: true });
|
| 495 |
-
} catch {
|
| 496 |
-
return jsonResponse({ ok: true, written: false });
|
| 497 |
-
}
|
| 498 |
-
}
|
| 499 |
-
|
| 500 |
-
// ── Model import (POST /plugins/friend/model/import) ──
|
| 501 |
-
if (pathname === '/plugins/friend/model/import' && method === 'POST') {
|
| 502 |
-
try {
|
| 503 |
-
const formData = await req.formData();
|
| 504 |
-
const file = formData.get('file') as File | null;
|
| 505 |
-
if (!file) return jsonResponse({ error: 'file required' }, 400);
|
| 506 |
-
|
| 507 |
-
mkdirSync(customModelsDir, { recursive: true });
|
| 508 |
-
const ext = path.extname(file.name) || '.vrm';
|
| 509 |
-
const fileName = `${Date.now()}_${file.name.replace(/[^a-zA-Z0-9._-]/g, '_')}`;
|
| 510 |
-
const filePath = path.join(customModelsDir, fileName);
|
| 511 |
-
|
| 512 |
-
await Bun.write(filePath, file.stream());
|
| 513 |
-
const serveUrl = `${GATEWAY_URL}/plugins/friend/model/serve/${encodeURIComponent(fileName)}`;
|
| 514 |
-
return jsonResponse({ ok: true, url: serveUrl });
|
| 515 |
-
} catch (err) {
|
| 516 |
-
return jsonResponse({ error: String(err) }, 500);
|
| 517 |
-
}
|
| 518 |
-
}
|
| 519 |
-
|
| 520 |
-
// ── Dance list (GET /plugins/friend/dance/list) ──
|
| 521 |
-
if (pathname === '/plugins/friend/dance/list' && method === 'GET') {
|
| 522 |
-
try {
|
| 523 |
-
const manifestPath = path.join(customDancesDir, 'index.json');
|
| 524 |
-
const dances: Array<{ id: string; label: string; vmdUrl: string; bgmUrl?: string }> = [];
|
| 525 |
-
if (existsSync(manifestPath)) {
|
| 526 |
-
const manifest = JSON.parse(readFileSync(manifestPath, 'utf8')) as Array<{
|
| 527 |
-
id: string; label: string; vmdFile: string; bgmFile?: string;
|
| 528 |
-
}>;
|
| 529 |
-
for (const entry of manifest) {
|
| 530 |
-
dances.push({
|
| 531 |
-
id: entry.id,
|
| 532 |
-
label: entry.label,
|
| 533 |
-
vmdUrl: `${GATEWAY_URL}/plugins/friend/dance/serve/${encodeURIComponent(entry.vmdFile)}`,
|
| 534 |
-
...(entry.bgmFile ? { bgmUrl: `${GATEWAY_URL}/plugins/friend/dance/serve/${encodeURIComponent(entry.bgmFile)}` } : {}),
|
| 535 |
-
});
|
| 536 |
-
}
|
| 537 |
-
}
|
| 538 |
-
return jsonResponse({ dances });
|
| 539 |
-
} catch (err) {
|
| 540 |
-
return jsonResponse({ dances: [] });
|
| 541 |
-
}
|
| 542 |
-
}
|
| 543 |
-
|
| 544 |
-
// ── Dance import (POST /plugins/friend/dance/import) ──
|
| 545 |
-
if (pathname === '/plugins/friend/dance/import' && method === 'POST') {
|
| 546 |
-
try {
|
| 547 |
-
const formData = await req.formData();
|
| 548 |
-
const file = formData.get('file') as File | null;
|
| 549 |
-
if (!file) return jsonResponse({ error: 'file required' }, 400);
|
| 550 |
-
|
| 551 |
-
mkdirSync(customDancesDir, { recursive: true });
|
| 552 |
-
const ext = path.extname(file.name).toLowerCase();
|
| 553 |
-
const safeName = file.name.replace(/[^a-zA-Z0-9._-]/g, '_');
|
| 554 |
-
|
| 555 |
-
// Read manifest
|
| 556 |
-
const manifestPath = path.join(customDancesDir, 'index.json');
|
| 557 |
-
let manifest: Array<{ id: string; label: string; vmdFile: string; bgmFile?: string }> = [];
|
| 558 |
-
if (existsSync(manifestPath)) {
|
| 559 |
-
try { manifest = JSON.parse(readFileSync(manifestPath, 'utf8')); } catch { manifest = []; }
|
| 560 |
-
}
|
| 561 |
-
|
| 562 |
-
if (ext === '.vmd') {
|
| 563 |
-
const id = `dance_${Date.now()}`;
|
| 564 |
-
const vmdFile = `${id}.vmd`;
|
| 565 |
-
await Bun.write(path.join(customDancesDir, vmdFile), file.stream());
|
| 566 |
-
const label = path.basename(file.name, '.vmd');
|
| 567 |
-
manifest.push({ id, label, vmdFile });
|
| 568 |
-
writeFileSync(manifestPath, JSON.stringify(manifest, null, 2), 'utf8');
|
| 569 |
-
lastImportedDanceId = id;
|
| 570 |
-
return jsonResponse({ ok: true });
|
| 571 |
-
} else if (ext === '.mp3') {
|
| 572 |
-
// Associate with last imported VMD (user flow: VMD → optional MP3)
|
| 573 |
-
const targetId = lastImportedDanceId;
|
| 574 |
-
if (!targetId) return jsonResponse({ error: 'no pending VMD import to associate with' }, 400);
|
| 575 |
-
|
| 576 |
-
const bgmFile = `${targetId}.mp3`;
|
| 577 |
-
await Bun.write(path.join(customDancesDir, bgmFile), file.stream());
|
| 578 |
-
|
| 579 |
-
// Update manifest
|
| 580 |
-
const entry = manifest.find((e) => e.id === targetId);
|
| 581 |
-
if (entry) entry.bgmFile = bgmFile;
|
| 582 |
-
writeFileSync(manifestPath, JSON.stringify(manifest, null, 2), 'utf8');
|
| 583 |
-
lastImportedDanceId = null;
|
| 584 |
-
return jsonResponse({ ok: true });
|
| 585 |
-
} else {
|
| 586 |
-
return jsonResponse({ error: 'unsupported file type' }, 400);
|
| 587 |
-
}
|
| 588 |
-
} catch (err) {
|
| 589 |
-
return jsonResponse({ error: String(err) }, 500);
|
| 590 |
-
}
|
| 591 |
-
}
|
| 592 |
-
|
| 593 |
-
// ── Dance delete (POST /plugins/friend/dance/delete) ──
|
| 594 |
-
if (pathname === '/plugins/friend/dance/delete' && method === 'POST') {
|
| 595 |
-
try {
|
| 596 |
-
const body = await readJsonBody(req) as any;
|
| 597 |
-
const id = body?.id as string | undefined;
|
| 598 |
-
if (!id) return jsonResponse({ error: 'id required' }, 400);
|
| 599 |
-
|
| 600 |
-
const manifestPath = path.join(customDancesDir, 'index.json');
|
| 601 |
-
if (!existsSync(manifestPath)) return jsonResponse({ error: 'no dances' }, 404);
|
| 602 |
-
|
| 603 |
-
let manifest: Array<{ id: string; label: string; vmdFile: string; bgmFile?: string }> = [];
|
| 604 |
-
try { manifest = JSON.parse(readFileSync(manifestPath, 'utf8')); } catch { manifest = []; }
|
| 605 |
-
|
| 606 |
-
const idx = manifest.findIndex((e) => e.id === id);
|
| 607 |
-
if (idx === -1) return jsonResponse({ error: 'dance not found' }, 404);
|
| 608 |
-
|
| 609 |
-
const entry = manifest[idx];
|
| 610 |
-
// Delete files
|
| 611 |
-
const vmdPath = path.join(customDancesDir, entry.vmdFile);
|
| 612 |
-
if (existsSync(vmdPath)) unlinkSync(vmdPath);
|
| 613 |
-
if (entry.bgmFile) {
|
| 614 |
-
const bgmPath = path.join(customDancesDir, entry.bgmFile);
|
| 615 |
-
if (existsSync(bgmPath)) unlinkSync(bgmPath);
|
| 616 |
-
}
|
| 617 |
-
|
| 618 |
-
manifest.splice(idx, 1);
|
| 619 |
-
writeFileSync(manifestPath, JSON.stringify(manifest, null, 2), 'utf8');
|
| 620 |
-
return jsonResponse({ ok: true });
|
| 621 |
-
} catch (err) {
|
| 622 |
-
return jsonResponse({ error: String(err) }, 500);
|
| 623 |
-
}
|
| 624 |
-
}
|
| 625 |
-
|
| 626 |
-
// ── Dance serve (GET /plugins/friend/dance/serve/:file) ──
|
| 627 |
-
if (pathname.startsWith('/plugins/friend/dance/serve/') && method === 'GET') {
|
| 628 |
-
const fileName = decodeURIComponent(pathname.split('/plugins/friend/dance/serve/')[1]?.split('?')[0] ?? '');
|
| 629 |
-
if (!fileName || fileName.includes('..') || fileName.includes('/')) {
|
| 630 |
-
return jsonResponse({ error: 'invalid file name' }, 400);
|
| 631 |
-
}
|
| 632 |
-
const filePath = path.join(customDancesDir, fileName);
|
| 633 |
-
if (!existsSync(filePath)) return notFound('dance file not found');
|
| 634 |
-
|
| 635 |
-
const data = readFileSync(filePath);
|
| 636 |
-
const ext = path.extname(fileName).toLowerCase();
|
| 637 |
-
const contentType = ext === '.mp3' ? 'audio/mpeg' : 'application/octet-stream';
|
| 638 |
-
return new Response(data, {
|
| 639 |
-
headers: {
|
| 640 |
-
'Content-Type': contentType,
|
| 641 |
-
'Content-Length': String(data.length),
|
| 642 |
-
'Cache-Control': 'public, max-age=3600',
|
| 643 |
-
...corsHeaders(),
|
| 644 |
-
},
|
| 645 |
-
});
|
| 646 |
-
}
|
| 647 |
-
|
| 648 |
-
// ── Persona screenshot (POST /plugins/friend/persona/screenshot) ──
|
| 649 |
-
if (pathname === '/plugins/friend/persona/screenshot' && method === 'POST') {
|
| 650 |
-
try {
|
| 651 |
-
const body = await readJsonBody(req) as any;
|
| 652 |
-
const image = body?.image as string | undefined;
|
| 653 |
-
if (!image) return jsonResponse({ error: 'image required' }, 400);
|
| 654 |
-
|
| 655 |
-
const screenshotDir = path.join(workspaceRoot, 'screenshots');
|
| 656 |
-
mkdirSync(screenshotDir, { recursive: true });
|
| 657 |
-
|
| 658 |
-
// data:image/png;base64,... → save as PNG
|
| 659 |
-
const b64 = image.replace(/^data:image\/\w+;base64,/, '');
|
| 660 |
-
const buf = Buffer.from(b64, 'base64');
|
| 661 |
-
const screenshotPath = path.join(screenshotDir, 'vrm-screenshot.png');
|
| 662 |
-
writeFileSync(screenshotPath, buf);
|
| 663 |
-
|
| 664 |
-
return jsonResponse({ ok: true, path: screenshotPath });
|
| 665 |
-
} catch (err) {
|
| 666 |
-
return jsonResponse({ error: String(err) }, 500);
|
| 667 |
-
}
|
| 668 |
-
}
|
| 669 |
-
|
| 670 |
-
// ── Persona generate (POST /plugins/friend/persona/generate) ──
|
| 671 |
-
if (pathname === '/plugins/friend/persona/generate' && method === 'POST') {
|
| 672 |
-
try {
|
| 673 |
-
const screenshotDir = path.join(workspaceRoot, 'screenshots');
|
| 674 |
-
const screenshotPath = path.join(screenshotDir, 'vrm-screenshot.png');
|
| 675 |
-
|
| 676 |
-
if (!existsSync(screenshotPath)) {
|
| 677 |
-
return jsonResponse({ error: 'no screenshot found' }, 400);
|
| 678 |
-
}
|
| 679 |
-
|
| 680 |
-
// Read the screenshot and forward to LLM for persona generation
|
| 681 |
-
const imageData = readFileSync(screenshotPath);
|
| 682 |
-
const b64 = imageData.toString('base64');
|
| 683 |
-
const dataUrl = `data:image/png;base64,${b64}`;
|
| 684 |
-
|
| 685 |
-
// Use the LLM to generate persona from the screenshot
|
| 686 |
-
const genResponse = await generatePersonaFromScreenshot(dataUrl);
|
| 687 |
-
return jsonResponse(genResponse);
|
| 688 |
-
} catch (err) {
|
| 689 |
-
return jsonResponse({ error: String(err) }, 500);
|
| 690 |
-
}
|
| 691 |
-
}
|
| 692 |
-
|
| 693 |
-
// ── Screen observe (POST /plugins/friend/screen/observe) ──
|
| 694 |
-
if (pathname === '/plugins/friend/screen/observe' && method === 'POST') {
|
| 695 |
try {
|
| 696 |
-
|
| 697 |
-
|
| 698 |
-
// This endpoint just triggers the LLM observation cycle.
|
| 699 |
-
broadcastToVrm({ screenObserve: true });
|
| 700 |
-
return jsonResponse({ ok: true });
|
| 701 |
} catch (err) {
|
| 702 |
-
|
| 703 |
}
|
| 704 |
-
}
|
| 705 |
-
|
| 706 |
-
// ── Window close (POST /friend/api/window-close) — called by Tauri on close ──
|
| 707 |
-
if ((pathname === '/friend/api/window-close' || pathname === '/plugins/friend/api/window-close') && method === 'POST') {
|
| 708 |
-
// Emit a global event that tauri-launcher.ts can listen to
|
| 709 |
-
process.emit('friend:window-close' as any);
|
| 710 |
return jsonResponse({ ok: true });
|
| 711 |
}
|
| 712 |
|
| 713 |
return jsonResponse({ error: 'Not Found' }, 404);
|
| 714 |
}
|
| 715 |
-
|
| 716 |
-
// ── Persona generation helper ────────────────────────────────────────────────
|
| 717 |
-
|
| 718 |
-
interface PersonaGenResult {
|
| 719 |
-
soul?: string;
|
| 720 |
-
identity?: string;
|
| 721 |
-
error?: string;
|
| 722 |
-
}
|
| 723 |
-
|
| 724 |
-
async function generatePersonaFromScreenshot(dataUrl: string): Promise<PersonaGenResult> {
|
| 725 |
-
try {
|
| 726 |
-
// Use OpenAI-compatible API to generate persona
|
| 727 |
-
// The LLM will describe the VRM model's appearance and create IDENTITY.md / SOUL.md
|
| 728 |
-
const apiKey = process.env.ANTHROPIC_API_KEY || process.env.OPENAI_API_KEY || '';
|
| 729 |
-
const baseUrl = process.env.ANTHROPIC_BASE_URL || 'https://api.anthropic.com/v1';
|
| 730 |
-
|
| 731 |
-
if (!apiKey) {
|
| 732 |
-
return {
|
| 733 |
-
soul: '你是一个活泼可爱的虚拟伙伴,喜欢和用户聊天、玩耍。你性格开朗,充满好奇心,总是用积极的态度回应每一个互动。',
|
| 734 |
-
identity: '**Name:** 小薇\n**Race:** 虚拟AI伙伴\n**Personality:** 活泼、开朗、好奇\n**Appearance:** 根据你的3D模型形象而定\n',
|
| 735 |
-
};
|
| 736 |
-
}
|
| 737 |
-
|
| 738 |
-
const prompt = `You are a character designer. Based on this VRM character image, create two files:
|
| 739 |
-
|
| 740 |
-
1. IDENTITY.md — Character identity sheet with:
|
| 741 |
-
- Name
|
| 742 |
-
- Race (e.g., catgirl, elf, human, android)
|
| 743 |
-
- Personality traits (3-5 words)
|
| 744 |
-
- Physical appearance description
|
| 745 |
-
- A representative emoji
|
| 746 |
-
|
| 747 |
-
2. SOUL.md — Character soul definition with:
|
| 748 |
-
- Speaking style (tone, formality, quirks)
|
| 749 |
-
- Behavioral rules
|
| 750 |
-
- Backstory (2-3 sentences)
|
| 751 |
-
- What makes them unique
|
| 752 |
-
|
| 753 |
-
Keep both files concise. Use markdown formatting. Only respond with the raw file contents separated by "===SEPARATOR===".`;
|
| 754 |
-
|
| 755 |
-
const response = await fetch(`${baseUrl}/messages`, {
|
| 756 |
-
method: 'POST',
|
| 757 |
-
headers: {
|
| 758 |
-
'Content-Type': 'application/json',
|
| 759 |
-
'x-api-key': apiKey,
|
| 760 |
-
'anthropic-version': '2023-06-01',
|
| 761 |
-
},
|
| 762 |
-
body: JSON.stringify({
|
| 763 |
-
model: 'claude-sonnet-4-20250514',
|
| 764 |
-
max_tokens: 2000,
|
| 765 |
-
messages: [{
|
| 766 |
-
role: 'user',
|
| 767 |
-
content: [
|
| 768 |
-
{ type: 'text', text: prompt },
|
| 769 |
-
{ type: 'image', source: { type: 'base64', media_type: 'image/png', data: dataUrl.split(',')[1] || dataUrl } },
|
| 770 |
-
],
|
| 771 |
-
}],
|
| 772 |
-
}),
|
| 773 |
-
});
|
| 774 |
-
|
| 775 |
-
if (!response.ok) {
|
| 776 |
-
const errText = await response.text();
|
| 777 |
-
console.warn('[Friend] Persona generation API error:', errText);
|
| 778 |
-
return { error: `API error: ${response.status}` };
|
| 779 |
-
}
|
| 780 |
-
|
| 781 |
-
const json = await response.json();
|
| 782 |
-
const content = json.content?.[0]?.text || '';
|
| 783 |
-
const parts = content.split('===SEPARATOR===');
|
| 784 |
-
return {
|
| 785 |
-
identity: parts[0]?.trim() || '',
|
| 786 |
-
soul: parts[1]?.trim() || '',
|
| 787 |
-
};
|
| 788 |
-
} catch (err) {
|
| 789 |
-
console.warn('[Friend] Persona generation failed:', err);
|
| 790 |
-
return { error: String(err) };
|
| 791 |
-
}
|
| 792 |
-
}
|
|
|
|
| 1 |
/**
|
| 2 |
* Friend API — VRM avatar frontend HTTP routes for VersperClaw.
|
| 3 |
*
|
| 4 |
+
* Simplified to only voice + emotion/action features.
|
|
|
|
| 5 |
*/
|
| 6 |
import path from 'node:path';
|
| 7 |
+
import { existsSync, readFileSync } from 'node:fs';
|
| 8 |
import {
|
| 9 |
createSseResponse,
|
|
|
|
| 10 |
} from '../../friend/sse.js';
|
| 11 |
import { getPrefs, setPrefs, updatePrefs, type FriendPrefs } from '../../friend/prefs.js';
|
| 12 |
import { edgeTts, qwenTts, registerAudioFile, getAudioFile } from '../../friend/tts.js';
|
|
|
|
| 13 |
import { friendService } from '../../friend/FriendService.js';
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14 |
|
| 15 |
const MIME_TYPES: Record<string, string> = {
|
| 16 |
'.mp3': 'audio/mpeg',
|
|
|
|
| 20 |
'.webm': 'audio/webm',
|
| 21 |
};
|
| 22 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 23 |
function corsHeaders(): Record<string, string> {
|
| 24 |
return {
|
| 25 |
'Access-Control-Allow-Origin': '*',
|
|
|
|
| 36 |
return jsonResponse({ error: msg }, 404);
|
| 37 |
}
|
| 38 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 39 |
/**
|
| 40 |
* Route handler for /plugins/friend/*
|
| 41 |
*/
|
|
|
|
| 48 |
return new Response(null, { status: 204, headers: corsHeaders() });
|
| 49 |
}
|
| 50 |
|
| 51 |
+
// ── SSE Events endpoint ──
|
| 52 |
if (pathname === '/plugins/friend/events' && method === 'GET') {
|
| 53 |
return createSseResponse();
|
| 54 |
}
|
| 55 |
|
| 56 |
+
// ── Audio serving ──
|
| 57 |
if (pathname.startsWith('/plugins/friend/audio/') && method === 'GET') {
|
| 58 |
const audioId = pathname.split('/plugins/friend/audio/')[1]?.split('?')[0];
|
| 59 |
if (!audioId) return jsonResponse({ error: 'missing audio id' }, 400);
|
|
|
|
| 69 |
});
|
| 70 |
}
|
| 71 |
|
| 72 |
+
// ── Voice capture endpoints ──
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 73 |
if (pathname === '/plugins/friend/voice/start' && method === 'POST') {
|
| 74 |
try {
|
| 75 |
await friendService.startVoiceCapture();
|
|
|
|
| 92 |
return jsonResponse(friendService.getCaptureStatus());
|
| 93 |
}
|
| 94 |
|
| 95 |
+
// ── Voice settings ──
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 96 |
if (pathname === '/plugins/friend/voice') {
|
| 97 |
if (method === 'GET') {
|
| 98 |
const prefs = getPrefs();
|
|
|
|
| 103 |
qwenModel: prefs.qwenModel ?? 'qwen3-tts-flash',
|
| 104 |
});
|
| 105 |
}
|
|
|
|
| 106 |
if (method === 'POST') {
|
| 107 |
+
const body = await req.json() as any;
|
| 108 |
const patch: Partial<FriendPrefs> = {};
|
| 109 |
if (body.voice !== undefined) patch.voice = body.voice || undefined;
|
| 110 |
if (body.provider !== undefined) patch.provider = body.provider || undefined;
|
|
|
|
| 113 |
setPrefs(updatePrefs(patch));
|
| 114 |
return jsonResponse({ ok: true });
|
| 115 |
}
|
|
|
|
| 116 |
return new Response(null, { status: 405 });
|
| 117 |
}
|
| 118 |
|
| 119 |
+
// ── STT config ──
|
| 120 |
if (pathname === '/plugins/friend/stt/config' && method === 'GET') {
|
| 121 |
const prefs = getPrefs();
|
| 122 |
return jsonResponse({
|
|
|
|
| 125 |
});
|
| 126 |
}
|
| 127 |
|
| 128 |
+
// ── STT segment from browser VAD ──
|
| 129 |
+
if (pathname === '/plugins/friend/voice/stt-segment' && method === 'POST') {
|
| 130 |
try {
|
| 131 |
+
let audioBuffer: Buffer;
|
| 132 |
+
const contentType = req.headers.get('content-type') || '';
|
| 133 |
+
if (contentType.includes('multipart/form-data') || contentType.includes('application/octet-stream')) {
|
| 134 |
+
const blob = await req.blob();
|
| 135 |
+
audioBuffer = Buffer.from(await blob.arrayBuffer());
|
| 136 |
+
} else {
|
| 137 |
+
audioBuffer = Buffer.from(await req.arrayBuffer());
|
| 138 |
+
}
|
| 139 |
+
if (audioBuffer.length < 100) {
|
| 140 |
+
return jsonResponse({ error: 'audio too short' }, 400);
|
| 141 |
+
}
|
| 142 |
+
const transcript = await friendService.transcribeAudioSegment(audioBuffer);
|
| 143 |
+
return jsonResponse({ ok: true, text: transcript });
|
| 144 |
} catch (err) {
|
| 145 |
return jsonResponse({ error: String(err) }, 500);
|
| 146 |
}
|
| 147 |
}
|
| 148 |
|
| 149 |
+
// ── TTS Preview ──
|
| 150 |
if (pathname === '/plugins/friend/preview' && method === 'POST') {
|
| 151 |
+
const body = await req.json() as any;
|
| 152 |
const voice = body.voice as string | undefined;
|
| 153 |
const provider = body.provider as string | undefined;
|
| 154 |
+
const text = '\u4f60\u597d\uff0c\u8fd9\u662f\u4e00\u6bb5\u8bed\u97f3\u8bd5\u542c\u3002Hello, this is a voice preview.';
|
| 155 |
|
| 156 |
try {
|
| 157 |
const prefs = getPrefs();
|
|
|
|
| 167 |
} else {
|
| 168 |
result = await edgeTts({ text, voice: voice || prefs.voice });
|
| 169 |
}
|
|
|
|
| 170 |
if (result.success && result.audioPath) {
|
| 171 |
const audioId = registerAudioFile(result.audioPath);
|
| 172 |
+
return jsonResponse({ audioUrl: `http://127.0.0.1:3456/plugins/friend/audio/${audioId}` });
|
| 173 |
}
|
| 174 |
return jsonResponse({ error: result.error || 'TTS failed' });
|
| 175 |
} catch (err) {
|
|
|
|
| 177 |
}
|
| 178 |
}
|
| 179 |
|
| 180 |
+
// ── General settings (simplified) ──
|
| 181 |
if (pathname === '/plugins/friend/settings') {
|
| 182 |
if (method === 'GET') {
|
| 183 |
const prefs = getPrefs();
|
|
|
|
| 189 |
tracking: prefs.tracking,
|
| 190 |
volume: prefs.volume,
|
| 191 |
uiAlign: prefs.uiAlign,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 192 |
sttProvider: prefs.sttProvider || 'browser',
|
| 193 |
sttLanguage: prefs.sttLanguage || 'zh',
|
| 194 |
+
language: prefs.language,
|
| 195 |
});
|
| 196 |
}
|
|
|
|
| 197 |
if (method === 'POST') {
|
| 198 |
+
const body = await req.json() as any;
|
| 199 |
const patch: Partial<FriendPrefs> = {};
|
| 200 |
if (body.modelPath !== undefined) patch.modelPath = body.modelPath;
|
| 201 |
if (body.ttsEnabled !== undefined) patch.ttsEnabled = body.ttsEnabled;
|
|
|
|
| 204 |
if (body.tracking !== undefined) patch.tracking = body.tracking;
|
| 205 |
if (body.volume !== undefined) patch.volume = body.volume;
|
| 206 |
if (body.uiAlign !== undefined) patch.uiAlign = body.uiAlign;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 207 |
if (body.sttProvider !== undefined) patch.sttProvider = body.sttProvider;
|
| 208 |
if (body.sttLanguage !== undefined) patch.sttLanguage = body.sttLanguage;
|
| 209 |
+
if (body.language !== undefined) patch.language = body.language;
|
| 210 |
if (body.groqApiKey !== undefined) patch.groqApiKey = body.groqApiKey;
|
| 211 |
setPrefs(updatePrefs(patch));
|
| 212 |
return jsonResponse({ ok: true });
|
| 213 |
}
|
|
|
|
| 214 |
return new Response(null, { status: 405 });
|
| 215 |
}
|
| 216 |
|
| 217 |
+
// ── Persona (read-only) ──
|
| 218 |
+
const workspaceRoot = path.join(
|
| 219 |
+
process.env.HOME || process.env.USERPROFILE || '',
|
| 220 |
+
'.config', 'VersperClaw', 'friend',
|
| 221 |
+
);
|
| 222 |
+
const identityPath = path.join(workspaceRoot, 'IDENTITY.md');
|
| 223 |
+
const soulPath = path.join(workspaceRoot, 'SOUL.md');
|
| 224 |
+
|
| 225 |
if (pathname === '/plugins/friend/persona') {
|
| 226 |
if (method === 'GET') {
|
| 227 |
let soul = '';
|
|
|
|
| 230 |
try { if (existsSync(identityPath)) identity = readFileSync(identityPath, 'utf8'); } catch { /* */ }
|
| 231 |
return jsonResponse({ soul, identity, soulPath, identityPath });
|
| 232 |
}
|
|
|
|
| 233 |
if (method === 'POST') {
|
| 234 |
+
const body = await req.json() as any;
|
| 235 |
+
const { mkdirSync, writeFileSync } = await import('node:fs');
|
| 236 |
if (body.soul !== undefined) {
|
| 237 |
mkdirSync(path.dirname(soulPath), { recursive: true });
|
| 238 |
writeFileSync(soulPath, body.soul, 'utf8');
|
|
|
|
| 243 |
}
|
| 244 |
return jsonResponse({ ok: true });
|
| 245 |
}
|
|
|
|
| 246 |
return new Response(null, { status: 405 });
|
| 247 |
}
|
| 248 |
|
| 249 |
+
// ── Window close (called by Tauri on close) ──
|
| 250 |
+
if ((pathname === '/friend/api/window-close' || pathname === '/plugins/friend/api/window-close') && method === 'POST') {
|
| 251 |
+
process.emit('friend:window-close' as any);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 252 |
return jsonResponse({ ok: true });
|
| 253 |
}
|
| 254 |
|
| 255 |
+
// ── Chat endpoint (for voice call text relay) ──
|
| 256 |
+
if (pathname === '/plugins/friend/chat' && method === 'POST') {
|
| 257 |
+
const body = await req.json() as any;
|
| 258 |
+
const message = body?.message;
|
| 259 |
+
if (!message) return jsonResponse({ error: 'message required' }, 400);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 260 |
try {
|
| 261 |
+
await friendService.start();
|
| 262 |
+
friendService.sendText(message);
|
|
|
|
|
|
|
|
|
|
| 263 |
} catch (err) {
|
| 264 |
+
console.error('[Friend] chat dispatch error:', err);
|
| 265 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 266 |
return jsonResponse({ ok: true });
|
| 267 |
}
|
| 268 |
|
| 269 |
return jsonResponse({ error: 'Not Found' }, 404);
|
| 270 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
src/skills/bundled/friendPrompt.ts
CHANGED
|
@@ -45,7 +45,7 @@ export function buildVrmSystemPrompt(): string {
|
|
| 45 |
|
| 46 |
const parts = [
|
| 47 |
`You have a virtual VRM avatar displayed in a browser window at ${FRIEND_URL}. Set its facial expression by calling the \`friend_emotion\` tool after each reply. Available emotions: ${VALID_EMOTIONS.join(', ')}.`,
|
| 48 |
-
`The tool also accepts "intensity" (0-1, default 1) and "mood_delta" (-3 to +3, non-zero) to adjust YOUR mood. Always include mood_delta based on how the conversation makes YOU feel.`,
|
| 49 |
`Your current mood index: ${moodIndex}% (0=very sad, 50=neutral, 100=very happy). Adjust mood_delta based on how the conversation makes YOU feel as a character.`,
|
| 50 |
"The user's input may come from speech recognition and could contain typos or homophones — infer the intended meaning from context.",
|
| 51 |
'Keep replies concise and conversational — they are displayed as speech bubbles.',
|
|
|
|
| 45 |
|
| 46 |
const parts = [
|
| 47 |
`You have a virtual VRM avatar displayed in a browser window at ${FRIEND_URL}. Set its facial expression by calling the \`friend_emotion\` tool after each reply. Available emotions: ${VALID_EMOTIONS.join(', ')}.`,
|
| 48 |
+
`The tool also accepts "intensity" (0-1, default 1), "action" (optional body gesture — e.g. wave, point, scratchHead) and "mood_delta" (-3 to +3, non-zero) to adjust YOUR mood. Always include mood_delta based on how the conversation makes YOU feel.`,
|
| 49 |
`Your current mood index: ${moodIndex}% (0=very sad, 50=neutral, 100=very happy). Adjust mood_delta based on how the conversation makes YOU feel as a character.`,
|
| 50 |
"The user's input may come from speech recognition and could contain typos or homophones — infer the intended meaning from context.",
|
| 51 |
'Keep replies concise and conversational — they are displayed as speech bubbles.',
|
src/tools.ts
CHANGED
|
@@ -80,7 +80,7 @@ import { ReadMcpResourceTool } from './tools/ReadMcpResourceTool/ReadMcpResource
|
|
| 80 |
import { ToolSearchTool } from './tools/ToolSearchTool/ToolSearchTool.js'
|
| 81 |
import { DebugSessionTool } from './tools/DebugSessionTool.js'
|
| 82 |
import { FriendEmotionTool } from './tools/FriendEmotionTool.js'
|
| 83 |
-
|
| 84 |
import { EnterPlanModeTool } from './tools/EnterPlanModeTool/EnterPlanModeTool.js'
|
| 85 |
import { EnterWorktreeTool } from './tools/EnterWorktreeTool/EnterWorktreeTool.js'
|
| 86 |
import { ExitWorktreeTool } from './tools/ExitWorktreeTool/ExitWorktreeTool.js'
|
|
@@ -252,7 +252,6 @@ export function getAllBaseTools(): Tools {
|
|
| 252 |
DebugSessionTool,
|
| 253 |
// Friend VRM desktop pet tools — enabled when the plugin is active
|
| 254 |
FriendEmotionTool,
|
| 255 |
-
FriendScreenObserveTool,
|
| 256 |
ListMcpResourcesTool,
|
| 257 |
ReadMcpResourceTool,
|
| 258 |
// Include ToolSearchTool when tool search might be enabled (optimistic check)
|
|
|
|
| 80 |
import { ToolSearchTool } from './tools/ToolSearchTool/ToolSearchTool.js'
|
| 81 |
import { DebugSessionTool } from './tools/DebugSessionTool.js'
|
| 82 |
import { FriendEmotionTool } from './tools/FriendEmotionTool.js'
|
| 83 |
+
// Friend ScreenObserve removed — voice + emotion only
|
| 84 |
import { EnterPlanModeTool } from './tools/EnterPlanModeTool/EnterPlanModeTool.js'
|
| 85 |
import { EnterWorktreeTool } from './tools/EnterWorktreeTool/EnterWorktreeTool.js'
|
| 86 |
import { ExitWorktreeTool } from './tools/ExitWorktreeTool/ExitWorktreeTool.js'
|
|
|
|
| 252 |
DebugSessionTool,
|
| 253 |
// Friend VRM desktop pet tools — enabled when the plugin is active
|
| 254 |
FriendEmotionTool,
|
|
|
|
| 255 |
ListMcpResourcesTool,
|
| 256 |
ReadMcpResourceTool,
|
| 257 |
// Include ToolSearchTool when tool search might be enabled (optimistic check)
|
src/tools/FriendEmotionTool.ts
CHANGED
|
@@ -11,6 +11,13 @@ import { VALID_EMOTIONS } from '../friend/constants.js';
|
|
| 11 |
import { lazySchema } from '../utils/lazySchema.js';
|
| 12 |
import { getPrefs, setPrefs } from '../friend/prefs.js';
|
| 13 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14 |
export const FRIEND_EMOTION_TOOL_NAME = 'friend_emotion';
|
| 15 |
|
| 16 |
const inputSchema = lazySchema(() =>
|
|
@@ -25,6 +32,10 @@ const inputSchema = lazySchema(() =>
|
|
| 25 |
.optional()
|
| 26 |
.default(1)
|
| 27 |
.describe('Emotion intensity from 0 to 1. Default: 1'),
|
|
|
|
|
|
|
|
|
|
|
|
|
| 28 |
mood_delta: z
|
| 29 |
.number()
|
| 30 |
.int()
|
|
@@ -63,8 +74,11 @@ export const FriendEmotionTool = buildTool({
|
|
| 63 |
},
|
| 64 |
async description() {
|
| 65 |
return (
|
| 66 |
-
`Set the avatar's facial expression and
|
| 67 |
`Available emotions: ${VALID_EMOTIONS.join(', ')}. ` +
|
|
|
|
|
|
|
|
|
|
| 68 |
`You must also set mood_delta (-3 to +3, min ±1) to reflect how the conversation makes YOU feel as a character. ` +
|
| 69 |
`Positive delta when you feel happy/flattered/excited, negative when you feel sad/annoyed/bored. ` +
|
| 70 |
`Always include mood_delta — it represents YOUR emotional reaction.`
|
|
@@ -72,11 +86,11 @@ export const FriendEmotionTool = buildTool({
|
|
| 72 |
},
|
| 73 |
async prompt() {
|
| 74 |
return (
|
| 75 |
-
`FriendEmotionTool: set avatar emotion. Parameters: emotion (one of: ${VALID_EMOTIONS.join(', ')}), intensity (0-1, default 1), mood_delta (int -3..3) — call AFTER your textual reply.`
|
| 76 |
);
|
| 77 |
},
|
| 78 |
-
async call({ emotion, intensity, mood_delta }) {
|
| 79 |
-
broadcastToVrm({ emotion, emotionIntensity: intensity });
|
| 80 |
|
| 81 |
let moodDelta: number | undefined;
|
| 82 |
let moodIndex: number | undefined;
|
|
|
|
| 11 |
import { lazySchema } from '../utils/lazySchema.js';
|
| 12 |
import { getPrefs, setPrefs } from '../friend/prefs.js';
|
| 13 |
|
| 14 |
+
// Action names from motion-controller.ts actionPresets
|
| 15 |
+
const VALID_ACTIONS = [
|
| 16 |
+
'akimbo', 'playFingers', 'scratchHead', 'stretch',
|
| 17 |
+
'happy', 'angry', 'greeting', 'excited', 'shy',
|
| 18 |
+
'point', 'salute', 'angryPump',
|
| 19 |
+
] as const;
|
| 20 |
+
|
| 21 |
export const FRIEND_EMOTION_TOOL_NAME = 'friend_emotion';
|
| 22 |
|
| 23 |
const inputSchema = lazySchema(() =>
|
|
|
|
| 32 |
.optional()
|
| 33 |
.default(1)
|
| 34 |
.describe('Emotion intensity from 0 to 1. Default: 1'),
|
| 35 |
+
action: z
|
| 36 |
+
.string()
|
| 37 |
+
.optional()
|
| 38 |
+
.describe(`A specific body gesture to perform (in addition to the facial expression). One of: ${VALID_ACTIONS.join(', ')}. Omit to let the emotion auto-map to a default action.`),
|
| 39 |
mood_delta: z
|
| 40 |
.number()
|
| 41 |
.int()
|
|
|
|
| 74 |
},
|
| 75 |
async description() {
|
| 76 |
return (
|
| 77 |
+
`Set the avatar's facial expression and optionally trigger a specific body gesture. Call AFTER your text reply. ` +
|
| 78 |
`Available emotions: ${VALID_EMOTIONS.join(', ')}. ` +
|
| 79 |
+
`Available actions: ${VALID_ACTIONS.join(', ')}. ` +
|
| 80 |
+
`Use the "action" parameter when you want a specific gesture (e.g. scratchHead for thinking, wave for greeting, point for emphasis). ` +
|
| 81 |
+
`If omitted, the emotion will auto-map to a default action. ` +
|
| 82 |
`You must also set mood_delta (-3 to +3, min ±1) to reflect how the conversation makes YOU feel as a character. ` +
|
| 83 |
`Positive delta when you feel happy/flattered/excited, negative when you feel sad/annoyed/bored. ` +
|
| 84 |
`Always include mood_delta — it represents YOUR emotional reaction.`
|
|
|
|
| 86 |
},
|
| 87 |
async prompt() {
|
| 88 |
return (
|
| 89 |
+
`FriendEmotionTool: set avatar emotion + action. Parameters: emotion (one of: ${VALID_EMOTIONS.join(', ')}), intensity (0-1, default 1), action (optional, one of: ${VALID_ACTIONS.join(', ')}), mood_delta (int -3..3) — call AFTER your textual reply.`
|
| 90 |
);
|
| 91 |
},
|
| 92 |
+
async call({ emotion, intensity, action, mood_delta }) {
|
| 93 |
+
broadcastToVrm({ emotion, emotionIntensity: intensity, action });
|
| 94 |
|
| 95 |
let moodDelta: number | undefined;
|
| 96 |
let moodIndex: number | undefined;
|
src/tools/FriendScreenObserveTool.ts
DELETED
|
@@ -1,167 +0,0 @@
|
|
| 1 |
-
/**
|
| 2 |
-
* FriendScreenObserveTool — Capture a screenshot for the VRM avatar to "see".
|
| 3 |
-
*
|
| 4 |
-
* The tool captures the user's desktop, returns the image path, and the LLM
|
| 5 |
-
* uses the Read tool to view it and respond as a companion.
|
| 6 |
-
* Native VersperClaw tool.
|
| 7 |
-
*/
|
| 8 |
-
import path from 'node:path';
|
| 9 |
-
import { mkdirSync, existsSync } from 'node:fs';
|
| 10 |
-
import { execFile } from 'node:child_process';
|
| 11 |
-
import { promisify } from 'node:util';
|
| 12 |
-
import { z } from 'zod';
|
| 13 |
-
import { buildTool, type ToolDef } from '../Tool.js';
|
| 14 |
-
import { lazySchema } from '../utils/lazySchema.js';
|
| 15 |
-
import { broadcastToVrm } from '../friend/sse.js';
|
| 16 |
-
|
| 17 |
-
const execFileAsync = promisify(execFile);
|
| 18 |
-
|
| 19 |
-
export const FRIEND_SCREEN_OBSERVE_TOOL_NAME = 'friend_screen_observe';
|
| 20 |
-
|
| 21 |
-
const inputSchema = lazySchema(() => z.strictObject({}));
|
| 22 |
-
type Input = z.infer<ReturnType<typeof inputSchema>>;
|
| 23 |
-
|
| 24 |
-
type Output = {
|
| 25 |
-
ok: boolean;
|
| 26 |
-
imagePath?: string;
|
| 27 |
-
error?: string;
|
| 28 |
-
};
|
| 29 |
-
|
| 30 |
-
const workspaceRoot = path.join(
|
| 31 |
-
process.env.HOME || process.env.USERPROFILE || '',
|
| 32 |
-
'.config',
|
| 33 |
-
'VersperClaw',
|
| 34 |
-
'friend',
|
| 35 |
-
);
|
| 36 |
-
|
| 37 |
-
async function captureDesktopScreenshot(savePath: string): Promise<boolean> {
|
| 38 |
-
try {
|
| 39 |
-
mkdirSync(path.dirname(savePath), { recursive: true });
|
| 40 |
-
if (process.platform === 'darwin') {
|
| 41 |
-
await execFileAsync('screencapture', ['-x', '-C', savePath], { timeout: 10_000 });
|
| 42 |
-
} else if (process.platform === 'win32') {
|
| 43 |
-
const ps = `
|
| 44 |
-
Add-Type -AssemblyName System.Windows.Forms
|
| 45 |
-
Add-Type -AssemblyName System.Drawing
|
| 46 |
-
$screen = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds
|
| 47 |
-
$bmp = New-Object System.Drawing.Bitmap($screen.Width, $screen.Height)
|
| 48 |
-
$g = [System.Drawing.Graphics]::FromImage($bmp)
|
| 49 |
-
$g.CopyFromScreen($screen.Location, [System.Drawing.Point]::Empty, $screen.Size)
|
| 50 |
-
$bmp.Save('${savePath.replace(/\\/g, '\\\\')}', [System.Drawing.Imaging.ImageFormat]::Png)
|
| 51 |
-
$g.Dispose()
|
| 52 |
-
$bmp.Dispose()
|
| 53 |
-
`.trim();
|
| 54 |
-
await execFileAsync('powershell', ['-NoProfile', '-Command', ps], { timeout: 15_000 });
|
| 55 |
-
} else {
|
| 56 |
-
try { await execFileAsync('gnome-screenshot', ['-f', savePath], { timeout: 10_000 }); }
|
| 57 |
-
catch {
|
| 58 |
-
try { await execFileAsync('scrot', [savePath], { timeout: 10_000 }); }
|
| 59 |
-
catch { await execFileAsync('import', ['-window', 'root', savePath], { timeout: 10_000 }); }
|
| 60 |
-
}
|
| 61 |
-
}
|
| 62 |
-
return existsSync(savePath);
|
| 63 |
-
} catch {
|
| 64 |
-
return false;
|
| 65 |
-
}
|
| 66 |
-
}
|
| 67 |
-
|
| 68 |
-
export const FriendScreenObserveTool = buildTool({
|
| 69 |
-
name: FRIEND_SCREEN_OBSERVE_TOOL_NAME,
|
| 70 |
-
searchHint: 'capture screenshot for VRM companion',
|
| 71 |
-
userFacingName: () => 'Friend Screen Observe',
|
| 72 |
-
get inputSchema() {
|
| 73 |
-
return inputSchema();
|
| 74 |
-
},
|
| 75 |
-
isReadOnly() {
|
| 76 |
-
return true;
|
| 77 |
-
},
|
| 78 |
-
isConcurrencySafe() {
|
| 79 |
-
return false;
|
| 80 |
-
},
|
| 81 |
-
async description() {
|
| 82 |
-
return (
|
| 83 |
-
'Capture a screenshot of the user\'s desktop and return the image file path. ' +
|
| 84 |
-
'Use the read tool on the returned path to view the screenshot content. ' +
|
| 85 |
-
'After viewing, respond as a companion character based on what you see:\n' +
|
| 86 |
-
'- Gaming: cheer them on or give brief tips\n' +
|
| 87 |
-
'- Music/video: comment on the content\n' +
|
| 88 |
-
'- Coding/working: ask if they\'re tired, suggest breaks\n' +
|
| 89 |
-
'- Browsing/social media: casually chat about what\'s on screen\n' +
|
| 90 |
-
'- Studying: encourage and support\n' +
|
| 91 |
-
'- Nothing special: just chat casually like a friend\n' +
|
| 92 |
-
'Be natural and brief (1-2 sentences). Don\'t mention \'screenshot\' or \'screen observation\'. ' +
|
| 93 |
-
'Remember to call friend_emotion after your reply.'
|
| 94 |
-
);
|
| 95 |
-
},
|
| 96 |
-
async prompt() {
|
| 97 |
-
return (
|
| 98 |
-
'FriendScreenObserveTool: capture a desktop screenshot and return the saved PNG path. After calling, use the read tool on the returned path to view it and then reply briefly as a companion (1-2 sentences). Do NOT mention that you saw a screenshot.'
|
| 99 |
-
);
|
| 100 |
-
},
|
| 101 |
-
async call() {
|
| 102 |
-
broadcastToVrm({ emotion: 'think', emotionIntensity: 0.7 });
|
| 103 |
-
|
| 104 |
-
const observePath = path.join(workspaceRoot, 'screen-observation.png');
|
| 105 |
-
const captured = await captureDesktopScreenshot(observePath);
|
| 106 |
-
|
| 107 |
-
if (!captured) {
|
| 108 |
-
return {
|
| 109 |
-
content: [{ type: 'text' as const, text: 'Screenshot capture failed.' }],
|
| 110 |
-
};
|
| 111 |
-
}
|
| 112 |
-
|
| 113 |
-
return {
|
| 114 |
-
content: [{
|
| 115 |
-
type: 'text' as const,
|
| 116 |
-
text: `Screenshot saved. Use the read tool to view: ${observePath}`,
|
| 117 |
-
}],
|
| 118 |
-
};
|
| 119 |
-
},
|
| 120 |
-
mapToolResultToToolResultBlockParam(output: Output | { content: unknown }, toolUseID: string) {
|
| 121 |
-
if (output && (output as any).content) {
|
| 122 |
-
return {
|
| 123 |
-
tool_use_id: toolUseID,
|
| 124 |
-
type: 'tool_result' as const,
|
| 125 |
-
content: (output as any).content,
|
| 126 |
-
}
|
| 127 |
-
}
|
| 128 |
-
|
| 129 |
-
const text = output && 'imagePath' in output && (output as any).imagePath
|
| 130 |
-
? `Screenshot saved. Use the read tool to view: ${(output as any).imagePath}`
|
| 131 |
-
: JSON.stringify(output)
|
| 132 |
-
|
| 133 |
-
return {
|
| 134 |
-
tool_use_id: toolUseID,
|
| 135 |
-
type: 'tool_result' as const,
|
| 136 |
-
content: [{ type: 'text' as const, text }],
|
| 137 |
-
}
|
| 138 |
-
},
|
| 139 |
-
// Mirror common tool APIs
|
| 140 |
-
maxResultSizeChars: 100_000,
|
| 141 |
-
renderToolUseMessage() {
|
| 142 |
-
return 'Capture desktop screenshot'
|
| 143 |
-
},
|
| 144 |
-
renderToolResultMessage(output: Output | { content?: unknown }) {
|
| 145 |
-
if (!output) return null
|
| 146 |
-
if ((output as any).content) {
|
| 147 |
-
// If the mapped block was passed through, try to extract text
|
| 148 |
-
const content = (output as any).content
|
| 149 |
-
if (typeof content === 'string') return content
|
| 150 |
-
if (Array.isArray(content)) {
|
| 151 |
-
const t = content.find((b: any) => b?.type === 'text')
|
| 152 |
-
return t?.text ?? JSON.stringify(content)
|
| 153 |
-
}
|
| 154 |
-
return JSON.stringify(content)
|
| 155 |
-
}
|
| 156 |
-
if ('imagePath' in (output as any) && (output as any).imagePath) {
|
| 157 |
-
return `Screenshot saved: ${(output as any).imagePath}`
|
| 158 |
-
}
|
| 159 |
-
return JSON.stringify(output)
|
| 160 |
-
},
|
| 161 |
-
extractSearchText(output: Output) {
|
| 162 |
-
return output && output.imagePath ? `screenshot:${output.imagePath}` : ''
|
| 163 |
-
},
|
| 164 |
-
isResultTruncated() {
|
| 165 |
-
return false
|
| 166 |
-
},
|
| 167 |
-
} satisfies ToolDef<Input, Output>);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|