chenbhao commited on
Commit
a18aff5
Β·
1 Parent(s): d7e83a1

feat: cycle time activate check friend

Browse files
src/components/friend/frontend/App.tsx CHANGED
@@ -75,7 +75,7 @@ export default function App() {
75
  const [screenObserve, setScreenObserve] = useState(false)
76
  const [screenObserveInterval, setScreenObserveInterval] = useState(60)
77
  const [language, setLanguage] = useState<'zh' | 'en'>(() => navigator.language.startsWith('zh') ? 'zh' : 'en')
78
- const [sttProvider, setSttProvider] = useState<'browser' | 'anthropic' | 'local' | 'doubao'>('browser')
79
  const t = (zh: string, en: string) => language === 'en' ? en : zh
80
  usePassThrough(!settingsOpen && !historyOpen)
81
 
 
75
  const [screenObserve, setScreenObserve] = useState(false)
76
  const [screenObserveInterval, setScreenObserveInterval] = useState(60)
77
  const [language, setLanguage] = useState<'zh' | 'en'>(() => navigator.language.startsWith('zh') ? 'zh' : 'en')
78
+ const [sttProvider, setSttProvider] = useState<'browser' | 'groq' | 'anthropic' | 'local' | 'doubao'>('browser')
79
  const t = (zh: string, en: string) => language === 'en' ? en : zh
80
  usePassThrough(!settingsOpen && !historyOpen)
81
 
src/components/friend/frontend/components/TextBubble.tsx CHANGED
@@ -1,4 +1,5 @@
1
  import { useEffect, useState, useRef, useCallback } from 'react'
 
2
  import { FRIEND_API } from '../api'
3
  import { LipSync } from '../lip-sync'
4
 
@@ -50,6 +51,7 @@ export function TextBubble({ onMessage, enabled = true, ttsEnabled = true }: { o
50
  const [thinking, setThinking] = useState(false)
51
  const [imageUrl, setImageUrl] = useState<string | null>(null)
52
  const [zoomedSrc, setZoomedSrc] = useState<string | null>(null)
 
53
  const timerRef = useRef<ReturnType<typeof setTimeout> | null>(null)
54
  const typewriterRef = useRef<ReturnType<typeof setInterval> | null>(null)
55
  const scrollRef = useRef<HTMLDivElement>(null)
@@ -90,10 +92,25 @@ export function TextBubble({ onMessage, enabled = true, ttsEnabled = true }: { o
90
  }
91
  }, [charCount])
92
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
93
  const hideBubble = useCallback(() => {
94
  setVisible(false)
95
  setText('')
96
  setCharCount(0)
 
97
  setImageUrl(null)
98
  setThinking(false)
99
  chars.current = []
@@ -443,11 +460,31 @@ export function TextBubble({ onMessage, enabled = true, ttsEnabled = true }: { o
443
  )}
444
  {text && (
445
  <div style={textStyle}>
446
- {chars.current.map((ch, i) => (
447
- i < charCount ? (
448
- <span key={i} style={popCharStyle}>{ch === '\n' ? <br /> : ch}</span>
449
- ) : null
450
- ))}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
451
  </div>
452
  )}
453
  </div>
 
1
  import { useEffect, useState, useRef, useCallback } from 'react'
2
+ import { marked } from 'marked'
3
  import { FRIEND_API } from '../api'
4
  import { LipSync } from '../lip-sync'
5
 
 
51
  const [thinking, setThinking] = useState(false)
52
  const [imageUrl, setImageUrl] = useState<string | null>(null)
53
  const [zoomedSrc, setZoomedSrc] = useState<string | null>(null)
54
+ const [renderedHtml, setRenderedHtml] = useState('')
55
  const timerRef = useRef<ReturnType<typeof setTimeout> | null>(null)
56
  const typewriterRef = useRef<ReturnType<typeof setInterval> | null>(null)
57
  const scrollRef = useRef<HTMLDivElement>(null)
 
92
  }
93
  }, [charCount])
94
 
95
+ // Render markdown when all text is revealed (typewriter complete)
96
+ useEffect(() => {
97
+ if (charCount >= chars.current.length && chars.current.length > 0) {
98
+ try {
99
+ const html = marked.parse(fullTextRef.current)
100
+ setRenderedHtml(typeof html === 'string' ? html : String(html))
101
+ } catch {
102
+ setRenderedHtml('')
103
+ }
104
+ } else if (!text) {
105
+ setRenderedHtml('')
106
+ }
107
+ }, [charCount, text])
108
+
109
  const hideBubble = useCallback(() => {
110
  setVisible(false)
111
  setText('')
112
  setCharCount(0)
113
+ setRenderedHtml('')
114
  setImageUrl(null)
115
  setThinking(false)
116
  chars.current = []
 
460
  )}
461
  {text && (
462
  <div style={textStyle}>
463
+ {renderedHtml && charCount >= chars.current.length ? (
464
+ <>
465
+ <style>{`
466
+ .rendered-markdown p { margin: 0.3em 0; }
467
+ .rendered-markdown ul, .rendered-markdown ol { margin: 0.3em 0; padding-left: 1.5em; }
468
+ .rendered-markdown li { margin: 0.1em 0; }
469
+ .rendered-markdown code { background: rgba(255,255,255,0.12); padding: 1px 5px; border-radius: 4px; font-size: 0.85em; }
470
+ .rendered-markdown pre { background: rgba(0,0,0,0.35); padding: 10px; border-radius: 8px; overflow-x: auto; margin: 0.4em 0; }
471
+ .rendered-markdown pre code { background: none; padding: 0; border-radius: 0; }
472
+ .rendered-markdown strong { color: rgba(160,200,255,0.9); }
473
+ .rendered-markdown em { color: rgba(200,200,255,0.8); }
474
+ .rendered-markdown h1, .rendered-markdown h2, .rendered-markdown h3, .rendered-markdown h4 { margin: 0.4em 0 0.2em; }
475
+ .rendered-markdown a { color: rgba(100,160,255,0.9); text-decoration: underline; }
476
+ .rendered-markdown blockquote { border-left: 3px solid rgba(100,160,255,0.5); padding-left: 10px; margin: 0.4em 0; color: rgba(255,255,255,0.7); }
477
+ .rendered-markdown hr { border: none; border-top: 1px solid rgba(255,255,255,0.15); margin: 0.5em 0; }
478
+ `}</style>
479
+ <div dangerouslySetInnerHTML={{ __html: renderedHtml }} className="rendered-markdown" />
480
+ </>
481
+ ) : (
482
+ chars.current.map((ch, i) => (
483
+ i < charCount ? (
484
+ <span key={i} style={popCharStyle}>{ch === '\n' ? <br /> : ch}</span>
485
+ ) : null
486
+ ))
487
+ )}
488
  </div>
489
  )}
490
  </div>
src/components/friend/frontend/hooks/useServerStt.ts CHANGED
@@ -1,5 +1,5 @@
1
  /**
2
- * Hook for using server-side STT (Anthropic Voice Stream, Local Whisper, Doubao)
3
  * in the Friend Tauri app.
4
  *
5
  * Audio capture is done server-side via cpal (in-process native addon)
@@ -8,7 +8,10 @@
8
  *
9
  * Two modes:
10
  * - push-to-talk: POST /voice/start β†’ capture β†’ POST /voice/stop β†’ get text
11
- * - streaming: POST /voice/start β†’ poll /voice/status β†’ POST /voice/stop
 
 
 
12
  */
13
  import { useRef, useCallback, useState } from 'react'
14
 
@@ -18,10 +21,13 @@ export type SttProvider = 'browser' | 'groq' | 'anthropic' | 'local' | 'doubao'
18
 
19
  export function useServerStt() {
20
  const [connected, setConnected] = useState(false)
 
 
 
21
 
22
  // ── Push-to-talk ──────────────────────────────────────────────────────
23
 
24
- /** Start push-to-talk: tell backend to start audio capture via cpal. */
25
  const startPushToTalk = useCallback(
26
  async (_provider: SttProvider, _language: string): Promise<void> => {
27
  const res = await fetch(`${FRIEND_API_BASE}/voice/start`, {
@@ -47,11 +53,48 @@ export function useServerStt() {
47
  return data.text || ''
48
  }, [])
49
 
50
- // ── Streaming (voice call) ────────────────────────────────────────────
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
51
 
52
  /**
53
  * Start streaming STT for voice call mode.
54
- * Transcripts arrive via polling /voice/status.
 
55
  */
56
  const startStreaming = useCallback(
57
  (
@@ -60,18 +103,26 @@ export function useServerStt() {
60
  onError: (err: string) => void,
61
  _language = 'zh',
62
  ) => {
63
- // Start capture
 
 
 
 
 
 
64
  fetch(`${FRIEND_API_BASE}/voice/start`, { method: 'POST' })
65
  .then(async (res) => {
 
66
  if (!res.ok) {
67
  const body = await res.json().catch(() => ({}))
68
  throw new Error(body.error || 'STT start failed')
69
  }
70
  setConnected(true)
71
 
72
- // Poll for interim results every 500ms
73
  const pollId = setInterval(async () => {
74
  try {
 
75
  const statusRes = await fetch(`${FRIEND_API_BASE}/voice/status`, {
76
  method: 'POST',
77
  })
@@ -84,30 +135,43 @@ export function useServerStt() {
84
  // poll failed, ignore
85
  }
86
  }, 500)
87
-
88
- // Store poll ID for cleanup
89
  ;(window as any).__friendSttPollId = pollId
 
 
 
 
 
 
90
  })
91
  .catch((err) => {
92
- onError(err.message)
 
 
93
  })
94
  },
95
- [],
96
  )
97
 
98
  /**
99
  * Stop streaming STT.
100
- * Returns the final transcript text from the server.
101
  */
102
  const stopStreaming = useCallback(async (): Promise<string> => {
103
- // Stop polling
 
104
  const pollId = (window as any).__friendSttPollId
105
  if (pollId) {
106
  clearInterval(pollId)
107
  delete (window as any).__friendSttPollId
108
  }
109
 
110
- // Stop capture and get final transcript
 
 
 
 
 
 
111
  const res = await fetch(`${FRIEND_API_BASE}/voice/stop`, { method: 'POST' })
112
  setConnected(false)
113
  if (!res.ok) return ''
 
1
  /**
2
+ * Hook for using server-side STT (Groq Whisper, Anthropic Voice Stream, Local Whisper, Doubao)
3
  * in the Friend Tauri app.
4
  *
5
  * Audio capture is done server-side via cpal (in-process native addon)
 
8
  *
9
  * Two modes:
10
  * - push-to-talk: POST /voice/start β†’ capture β†’ POST /voice/stop β†’ get text
11
+ * - streaming: POST /voice/start β†’ poll /voice/status + auto-cycle every 3s
12
+ *
13
+ * The auto-cycle in streaming mode ensures batch STT providers (Groq, local Whisper)
14
+ * produce regular final transcripts even though they don't support interim results.
15
  */
16
  import { useRef, useCallback, useState } from 'react'
17
 
 
21
 
22
  export function useServerStt() {
23
  const [connected, setConnected] = useState(false)
24
+ const cycleTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null)
25
+ const cycleAccumRef = useRef('')
26
+ const streamingGenRef = useRef(0) // generation counter to prevent stale callbacks
27
 
28
  // ── Push-to-talk ──────────────────────────────────────────────────────
29
 
30
+ /** Start push-to-talk: tell backend to start audio capture. */
31
  const startPushToTalk = useCallback(
32
  async (_provider: SttProvider, _language: string): Promise<void> => {
33
  const res = await fetch(`${FRIEND_API_BASE}/voice/start`, {
 
53
  return data.text || ''
54
  }, [])
55
 
56
+ // ── Streaming (voice call) with auto-cycle ────────────────────────────
57
+
58
+ const doCycle = useCallback(
59
+ (onTranscript: (text: string, isFinal: boolean) => void, gen: number): void => {
60
+ const isStale = () => streamingGenRef.current !== gen
61
+ fetch(`${FRIEND_API_BASE}/voice/stop`, { method: 'POST' })
62
+ .then(async (res) => {
63
+ if (isStale()) return
64
+ if (!res.ok) return
65
+ const data = await res.json()
66
+ if (data.text?.trim()) {
67
+ cycleAccumRef.current += data.text
68
+ onTranscript(cycleAccumRef.current, true)
69
+ }
70
+ })
71
+ .then(() => {
72
+ if (isStale()) return
73
+ // Restart capture for next cycle
74
+ return fetch(`${FRIEND_API_BASE}/voice/start`, { method: 'POST' })
75
+ })
76
+ .then(async (res) => {
77
+ if (isStale()) return
78
+ if (!res?.ok) {
79
+ setConnected(false)
80
+ return
81
+ }
82
+ setConnected(true)
83
+ // Schedule next cycle in 3s
84
+ cycleTimerRef.current = setTimeout(
85
+ () => doCycle(onTranscript, gen),
86
+ 3000,
87
+ )
88
+ })
89
+ .catch(() => {})
90
+ },
91
+ [],
92
+ )
93
 
94
  /**
95
  * Start streaming STT for voice call mode.
96
+ * Polls interim results every 500ms AND auto-cycles every 3s to get
97
+ * final transcripts from batch STT providers (Groq, local Whisper).
98
  */
99
  const startStreaming = useCallback(
100
  (
 
103
  onError: (err: string) => void,
104
  _language = 'zh',
105
  ) => {
106
+ // Clean up any existing state and bump generation
107
+ cycleAccumRef.current = ''
108
+ const myGen = ++streamingGenRef.current
109
+ const existingPollId = (window as any).__friendSttPollId
110
+ if (existingPollId) { clearInterval(existingPollId); delete (window as any).__friendSttPollId }
111
+ if (cycleTimerRef.current) { clearTimeout(cycleTimerRef.current); cycleTimerRef.current = null }
112
+
113
  fetch(`${FRIEND_API_BASE}/voice/start`, { method: 'POST' })
114
  .then(async (res) => {
115
+ if (streamingGenRef.current !== myGen) return
116
  if (!res.ok) {
117
  const body = await res.json().catch(() => ({}))
118
  throw new Error(body.error || 'STT start failed')
119
  }
120
  setConnected(true)
121
 
122
+ // Poll for interim results every 500ms (works with streaming STT providers)
123
  const pollId = setInterval(async () => {
124
  try {
125
+ if (streamingGenRef.current !== myGen) { clearInterval(pollId); return }
126
  const statusRes = await fetch(`${FRIEND_API_BASE}/voice/status`, {
127
  method: 'POST',
128
  })
 
135
  // poll failed, ignore
136
  }
137
  }, 500)
 
 
138
  ;(window as any).__friendSttPollId = pollId
139
+
140
+ // Schedule first auto-cycle in 3s (for batch STT providers like Groq)
141
+ cycleTimerRef.current = setTimeout(
142
+ () => doCycle(onTranscript, myGen),
143
+ 3000,
144
+ )
145
  })
146
  .catch((err) => {
147
+ if (streamingGenRef.current === myGen) {
148
+ onError(err.message)
149
+ }
150
  })
151
  },
152
+ [doCycle],
153
  )
154
 
155
  /**
156
  * Stop streaming STT.
157
+ * Cleans up poll interval and auto-cycle, then returns the final transcript.
158
  */
159
  const stopStreaming = useCallback(async (): Promise<string> => {
160
+ // Stop all activity by bumping generation counter
161
+ streamingGenRef.current++
162
  const pollId = (window as any).__friendSttPollId
163
  if (pollId) {
164
  clearInterval(pollId)
165
  delete (window as any).__friendSttPollId
166
  }
167
 
168
+ // Stop auto-cycle
169
+ if (cycleTimerRef.current) {
170
+ clearTimeout(cycleTimerRef.current)
171
+ cycleTimerRef.current = null
172
+ }
173
+ cycleAccumRef.current = ''
174
+
175
  const res = await fetch(`${FRIEND_API_BASE}/voice/stop`, { method: 'POST' })
176
  setConnected(false)
177
  if (!res.ok) return ''
src/components/friend/frontend/package-lock.json CHANGED
@@ -12,6 +12,7 @@
12
  "@pixiv/three-vrm-animation": "^3.5.0",
13
  "@pixiv/three-vrm-core": "^3.5.1",
14
  "lucide-react": "^0.577.0",
 
15
  "mmd-parser": "^1.0.4",
16
  "react": "^18.3.1",
17
  "react-dom": "^18.3.1",
@@ -2005,6 +2006,18 @@
2005
  "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0"
2006
  }
2007
  },
 
 
 
 
 
 
 
 
 
 
 
 
2008
  "node_modules/meshoptimizer": {
2009
  "version": "0.18.1",
2010
  "resolved": "https://registry.npmmirror.com/meshoptimizer/-/meshoptimizer-0.18.1.tgz",
 
12
  "@pixiv/three-vrm-animation": "^3.5.0",
13
  "@pixiv/three-vrm-core": "^3.5.1",
14
  "lucide-react": "^0.577.0",
15
+ "marked": "^18.0.5",
16
  "mmd-parser": "^1.0.4",
17
  "react": "^18.3.1",
18
  "react-dom": "^18.3.1",
 
2006
  "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0"
2007
  }
2008
  },
2009
+ "node_modules/marked": {
2010
+ "version": "18.0.5",
2011
+ "resolved": "https://registry.npmmirror.com/marked/-/marked-18.0.5.tgz",
2012
+ "integrity": "sha512-S6GcvALHg6K4ohtu4E7x0a1AqhAjp6cV8KhLSyN9qVapnzJkusVBxZRcIU9AeYsbe6P1hKDusSbEOzGyyuce6w==",
2013
+ "license": "MIT",
2014
+ "bin": {
2015
+ "marked": "bin/marked.js"
2016
+ },
2017
+ "engines": {
2018
+ "node": ">= 20"
2019
+ }
2020
+ },
2021
  "node_modules/meshoptimizer": {
2022
  "version": "0.18.1",
2023
  "resolved": "https://registry.npmmirror.com/meshoptimizer/-/meshoptimizer-0.18.1.tgz",
src/components/friend/frontend/package.json CHANGED
@@ -12,6 +12,7 @@
12
  "@pixiv/three-vrm-animation": "^3.5.0",
13
  "@pixiv/three-vrm-core": "^3.5.1",
14
  "lucide-react": "^0.577.0",
 
15
  "mmd-parser": "^1.0.4",
16
  "react": "^18.3.1",
17
  "react-dom": "^18.3.1",
 
12
  "@pixiv/three-vrm-animation": "^3.5.0",
13
  "@pixiv/three-vrm-core": "^3.5.1",
14
  "lucide-react": "^0.577.0",
15
+ "marked": "^18.0.5",
16
  "mmd-parser": "^1.0.4",
17
  "react": "^18.3.1",
18
  "react-dom": "^18.3.1",