Enrique Cardoza commited on
Commit
b116637
·
1 Parent(s): ce89e4f

feat(tools): add text-to-speech capability to agent

Browse files

- Implement TTS API integration with audio/wav response handling
- Create agentic decision-making to identify text-to-speech requests
- Add new function to extract text to convert from user messages
- Implement audio file saving and display in chat interface
- Add proper error handling for TTS generation process
- Update system prompts to include both transcription and TTS capabilities
- Create dedicated directory for storing generated audio files
- Enhance UI messaging and placeholder text to reflect new capability
- Improve chat flow with follow-up responses after audio generation
- Prioritize TTS evaluation before transcription in the decision flow

Files changed (1) hide show
  1. app.py +228 -20
app.py CHANGED
@@ -2,6 +2,11 @@ import gradio as gr
2
  import anthropic
3
  import os
4
  import json
 
 
 
 
 
5
  from dotenv import load_dotenv
6
  from smolagents.mcp_client import MCPClient
7
  from gradio import ChatMessage
@@ -12,6 +17,7 @@ load_dotenv()
12
  # Get API keys from environment or use None as default
13
  anthropic_api_key = os.environ.get("ANTHROPIC_API_KEY")
14
  groq_api_key = os.environ.get("GROQ_API_KEY")
 
15
 
16
  # Initialize Anthropic client (will be updated if user provides a key)
17
  client = anthropic.Anthropic(api_key=anthropic_api_key) if anthropic_api_key else None
@@ -19,6 +25,9 @@ client = anthropic.Anthropic(api_key=anthropic_api_key) if anthropic_api_key els
19
  # MCP connection details
20
  mcp_url = "https://agents-mcp-hackathon-transcript-generator.hf.space/gradio_api/mcp/sse"
21
 
 
 
 
22
  # Global variable to store tools
23
  mcp_tools = {}
24
  tool_descriptions = {}
@@ -118,6 +127,44 @@ def process_transcription_from_url(audio_url, user_groq_api_key):
118
  print(traceback.format_exc())
119
  return f"Error: {str(e)}"
120
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
121
  # Extract URL from message
122
  def extract_url(message):
123
  import re
@@ -128,7 +175,7 @@ def extract_url(message):
128
  return match.group(0)
129
  return None
130
 
131
- # Function to decide if tool should be used
132
  def should_use_transcription_tool(prompt, available_tools):
133
  try:
134
  if not client:
@@ -191,6 +238,69 @@ def should_use_transcription_tool(prompt, available_tools):
191
  # Default to not using tool if there's an error
192
  return False, f"Error: {str(e)}"
193
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
194
  # Main chat function (streaming, with tool usage)
195
  def chat_with_tools(user_message, history, user_groq_api_key, user_anthropic_api_key):
196
  global client, anthropic_api_key, groq_api_key, tool_descriptions
@@ -220,20 +330,110 @@ def chat_with_tools(user_message, history, user_groq_api_key, user_anthropic_api
220
  # 1. "Thinking" phase
221
  yield history + [ChatMessage(role="assistant", content="Let me think...", metadata={"title": "🧠 Thinking", "status": "pending"})]
222
 
223
- # Extract URL from message
224
  audio_url = extract_url(user_message)
225
 
226
- # 2. Decision phase - Ask Claude to decide if we should use the transcription tool
227
- use_tool = False
228
- reasoning = ""
229
 
230
- if audio_url:
231
- use_tool, reasoning = should_use_transcription_tool(user_message, tool_descriptions)
232
- print(f"Tool decision: {use_tool}, Reasoning: {reasoning}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
233
 
234
- # 3. Tool usage phase - Process transcription if decided to use tool
235
- transcription_result = None
236
- if use_tool and audio_url:
 
 
 
 
 
 
237
  try:
238
  # Check if GROQ API key is available (either from env or user input)
239
  current_groq_api_key = groq_api_key if groq_api_key else user_groq_api_key
@@ -319,7 +519,7 @@ def chat_with_tools(user_message, history, user_groq_api_key, user_anthropic_api
319
  ))
320
  yield history
321
  return
322
- elif use_tool and not audio_url:
323
  # User wants transcription but didn't provide URL
324
  history.append(ChatMessage(
325
  role="assistant",
@@ -332,20 +532,22 @@ def chat_with_tools(user_message, history, user_groq_api_key, user_anthropic_api
332
  # 4. Claude response phase - for regular chat or when tool isn't needed
333
  try:
334
  # Provide system prompt with tools information
335
- system_prompt = f"""You are a helpful assistant that can transcribe audio using specialized tools.
336
 
337
  Available tools:
338
  - Audio transcription: You can convert speech in audio files to text when users provide audio URLs
 
339
 
340
  Tool usage guidelines:
341
- - The system has already determined that the transcription tool should not be used for this message
342
  - If the user asks about transcribing audio but didn't provide a URL, tell them you need an audio URL
343
- - For other queries, respond normally without mentioning the transcription functionality unless relevant
 
344
 
345
  When responding to general questions:
346
  - Provide helpful and accurate information
347
  - Be concise and direct
348
- - If the user's message suggests they might want to transcribe audio later, you can mention this capability
349
  """
350
  # Debug message format
351
  print(f"Sending {len(formatted_messages)} messages to Claude")
@@ -376,14 +578,20 @@ initialize_tools()
376
 
377
  # Create the Gradio interface
378
  with gr.Blocks() as demo:
379
- gr.Markdown("# Claude 3.7 Sonnet Chat with Transcription Tools")
380
- gr.Markdown("Send a message with an audio URL to generate a transcript using the MCP server.")
 
 
 
 
 
381
  with gr.Row():
382
  with gr.Column(scale=4):
383
  chatbot = gr.Chatbot(
384
  type="messages",
385
  label="Claude Agent",
386
- height=600
 
387
  )
388
  with gr.Column(scale=1):
389
  # Display API key inputs only if environment variables are not set
@@ -407,7 +615,7 @@ with gr.Blocks() as demo:
407
 
408
  with gr.Row():
409
  msg = gr.Textbox(
410
- placeholder="Ask a question or share an audio URL to get a transcript...",
411
  show_label=False,
412
  container=False,
413
  scale=9
 
2
  import anthropic
3
  import os
4
  import json
5
+ import requests
6
+ import base64
7
+ import tempfile
8
+ import time
9
+ from pathlib import Path
10
  from dotenv import load_dotenv
11
  from smolagents.mcp_client import MCPClient
12
  from gradio import ChatMessage
 
17
  # Get API keys from environment or use None as default
18
  anthropic_api_key = os.environ.get("ANTHROPIC_API_KEY")
19
  groq_api_key = os.environ.get("GROQ_API_KEY")
20
+ tts_api_url_base = os.environ.get("TTS_API_URL")
21
 
22
  # Initialize Anthropic client (will be updated if user provides a key)
23
  client = anthropic.Anthropic(api_key=anthropic_api_key) if anthropic_api_key else None
 
25
  # MCP connection details
26
  mcp_url = "https://agents-mcp-hackathon-transcript-generator.hf.space/gradio_api/mcp/sse"
27
 
28
+ # TTS API endpoint
29
+ tts_api_url = tts_api_url_base+"/generate"
30
+
31
  # Global variable to store tools
32
  mcp_tools = {}
33
  tool_descriptions = {}
 
127
  print(traceback.format_exc())
128
  return f"Error: {str(e)}"
129
 
130
+ # Generate speech from text using the TTS API
131
+ def generate_speech_from_text(text):
132
+ try:
133
+ # Prepare the request data
134
+ payload = {"prompt": text}
135
+ headers = {"Content-Type": "application/json"}
136
+
137
+ print(f"Sending TTS request for text (length): {len(text)}")
138
+
139
+ # Make the POST request to the TTS API
140
+ response = requests.post(tts_api_url, json=payload, headers=headers)
141
+
142
+ # Check if the request was successful
143
+ if response.status_code == 200:
144
+ print(f"TTS response received. Content-Type: {response.headers.get('Content-Type')}")
145
+
146
+ # Ensure the output directory exists
147
+ output_dir = Path("generated_audio")
148
+ output_dir.mkdir(exist_ok=True)
149
+
150
+ # Create a file to store the audio with a timestamp to avoid conflicts
151
+ timestamp = int(time.time())
152
+ audio_file_path = output_dir / f"speech_{timestamp}.wav"
153
+
154
+ # Write the audio bytes to the file
155
+ with open(audio_file_path, "wb") as f:
156
+ f.write(response.content)
157
+
158
+ print(f"Audio saved to file: {audio_file_path}")
159
+ return str(audio_file_path)
160
+ else:
161
+ return f"Error: API returned status code {response.status_code} - {response.text}"
162
+ except Exception as e:
163
+ import traceback
164
+ print(f"TTS error: {e}")
165
+ print(traceback.format_exc())
166
+ return f"Error: {str(e)}"
167
+
168
  # Extract URL from message
169
  def extract_url(message):
170
  import re
 
175
  return match.group(0)
176
  return None
177
 
178
+ # Function to decide if transcription tool should be used
179
  def should_use_transcription_tool(prompt, available_tools):
180
  try:
181
  if not client:
 
238
  # Default to not using tool if there's an error
239
  return False, f"Error: {str(e)}"
240
 
241
+ # Function to decide if TTS tool should be used
242
+ def should_use_tts_tool(prompt):
243
+ try:
244
+ if not client:
245
+ print("No Claude client available for TTS tool decision")
246
+ # Default to not using tool since we can't ask Claude
247
+ return False, "No Claude client available for decision", ""
248
+
249
+ system_prompt = """You are an AI assistant deciding whether to use a text-to-speech (TTS) tool.
250
+ You have access to a TTS tool that can generate speech audio from text.
251
+
252
+ You should ONLY respond with a pure JSON object that includes:
253
+ 1. "use_tool": true/false - whether to use the TTS tool
254
+ 2. "reasoning": brief explanation of your decision
255
+ 3. "text_to_convert": If use_tool is true, include the text that should be converted to speech.
256
+ Extract this from the user's message or generate appropriate text based on their request.
257
+
258
+ It should start with a '{' and end with a '}'.
259
+
260
+ Respond with true ONLY if:
261
+ - The user is clearly asking to generate speech or audio from text
262
+ - OR the user wants text read aloud
263
+ - OR the user wants to create a voiceover or narration
264
+ - OR the user explicitly mentions text-to-speech or TTS
265
+
266
+ When deciding what text to convert:
267
+ - If the user provides specific text in quotes or after phrases like "convert this to speech:", use that exact text
268
+ - If the user asks to convert their previous message, use that as the text
269
+ - If the user doesn't specify text but clearly wants TTS, ask them to provide the text
270
+
271
+ Respond with false if:
272
+ - The user is just chatting or asking questions not related to speech generation
273
+ - The user is asking about transcription (speech-to-text) instead
274
+ """
275
+
276
+ # Ensure we're sending a valid message
277
+ response = client.messages.create(
278
+ model="claude-3-7-sonnet-latest",
279
+ max_tokens=500,
280
+ system=system_prompt,
281
+ messages=[{"role": "user", "content": f"User message: {prompt}"}]
282
+ )
283
+
284
+ response_content = response.content[0].text
285
+ print(f"TTS tool decision response: {response_content}")
286
+
287
+ # Try to parse the JSON response
288
+ try:
289
+ result = json.loads(response_content)
290
+ text_to_convert = result.get("text_to_convert", "")
291
+ return result.get("use_tool", False), result.get("reasoning", "No reasoning provided"), text_to_convert
292
+ except json.JSONDecodeError:
293
+ print(f"Failed to parse JSON response: {response_content}")
294
+ # Fallback: check if the response contains "true"
295
+ return "true" in response_content.lower(), "Failed to parse JSON response", ""
296
+
297
+ except Exception as e:
298
+ import traceback
299
+ print(f"Error in TTS tool decision: {e}")
300
+ print(traceback.format_exc())
301
+ # Default to not using tool if there's an error
302
+ return False, f"Error: {str(e)}", ""
303
+
304
  # Main chat function (streaming, with tool usage)
305
  def chat_with_tools(user_message, history, user_groq_api_key, user_anthropic_api_key):
306
  global client, anthropic_api_key, groq_api_key, tool_descriptions
 
330
  # 1. "Thinking" phase
331
  yield history + [ChatMessage(role="assistant", content="Let me think...", metadata={"title": "🧠 Thinking", "status": "pending"})]
332
 
333
+ # Extract URL from message for transcription
334
  audio_url = extract_url(user_message)
335
 
336
+ # 2. Decision phase - First check if we should use the TTS tool
337
+ use_tts, tts_reasoning, text_to_convert = should_use_tts_tool(user_message)
 
338
 
339
+ # 3a. Tool usage phase - Process TTS if decided to use that tool
340
+ if use_tts:
341
+ try:
342
+ # Show tool call status
343
+ yield history + [ChatMessage(
344
+ role="assistant",
345
+ content=f"Generating speech from text...",
346
+ metadata={"title": "🔊 Text-to-Speech", "status": "pending"}
347
+ )]
348
+
349
+ # If no text was provided for conversion, ask the user
350
+ if not text_to_convert:
351
+ history.append(ChatMessage(
352
+ role="assistant",
353
+ content="I'd be happy to generate speech audio for you! Could you please provide the text you want me to convert to speech?",
354
+ metadata={"title": "ℹ️ Info", "status": "done"}
355
+ ))
356
+ yield history
357
+ return
358
+
359
+ # Process text-to-speech
360
+ audio_file_path = generate_speech_from_text(text_to_convert)
361
+
362
+ if audio_file_path and isinstance(audio_file_path, str) and audio_file_path.startswith("Error:"):
363
+ history.append(ChatMessage(
364
+ role="assistant",
365
+ content=audio_file_path,
366
+ metadata={"title": "❌ Error", "status": "done"}
367
+ ))
368
+ yield history
369
+ return
370
+ elif audio_file_path:
371
+ # Add the audio component message
372
+ audio_display_message = f"""I've generated speech audio from your text. You can play it below:
373
+
374
+ <audio controls src="/file={audio_file_path}" style="width: 100%;"></audio>
375
+
376
+ The text that was converted: "{text_to_convert}"
377
+ """
378
+ history.append(ChatMessage(
379
+ role="assistant",
380
+ content=audio_display_message,
381
+ metadata={"title": "🔊 Speech Generated", "status": "done"}
382
+ ))
383
+
384
+ yield history
385
+
386
+ # Let Claude provide a response about the generated audio
387
+ try:
388
+ # Create a prompt for Claude to respond about the TTS generation
389
+ claude_prompt = f"You've just generated speech audio for the user from this text: \"{text_to_convert}\". Please provide a brief, helpful response. You might ask if the audio meets their needs or if they'd like to convert more text."
390
+
391
+ # Send request to Claude
392
+ response = client.messages.create(
393
+ model="claude-3-7-sonnet-latest",
394
+ max_tokens=1024,
395
+ system="You are a helpful assistant that can convert text to speech using specialized tools. When responding to the user about generated speech, be brief and helpful. Don't repeat the full text that was converted unless it's very short.",
396
+ messages=[{"role": "user", "content": claude_prompt}]
397
+ )
398
+
399
+ # Add Claude's response to history
400
+ response_content = response.content[0].text
401
+ history.append(ChatMessage(role="assistant", content=response_content))
402
+ yield history
403
+ return
404
+ except Exception as e:
405
+ print(f"Error generating response about TTS: {str(e)}")
406
+ # Continue without Claude's additional response
407
+ else:
408
+ history.append(ChatMessage(
409
+ role="assistant",
410
+ content="I wasn't able to generate speech from your text. Please try again with different text.",
411
+ metadata={"title": "❌ Error", "status": "done"}
412
+ ))
413
+ yield history
414
+ return
415
+ except Exception as e:
416
+ import traceback
417
+ error_details = traceback.format_exc()
418
+ print(f"TTS error details: {error_details}")
419
+
420
+ history.append(ChatMessage(
421
+ role="assistant",
422
+ content=f"Error generating speech: {str(e)}",
423
+ metadata={"title": "❌ Error", "status": "done"}
424
+ ))
425
+ yield history
426
+ return
427
 
428
+ # 2b. If not using TTS, check if we should use the transcription tool
429
+ if not use_tts and audio_url:
430
+ use_transcription, transcription_reasoning = should_use_transcription_tool(user_message, tool_descriptions)
431
+ print(f"Tool decision: {use_transcription}, Reasoning: {transcription_reasoning}")
432
+ else:
433
+ use_transcription = False
434
+
435
+ # 3b. Process transcription if decided to use that tool
436
+ if use_transcription and audio_url:
437
  try:
438
  # Check if GROQ API key is available (either from env or user input)
439
  current_groq_api_key = groq_api_key if groq_api_key else user_groq_api_key
 
519
  ))
520
  yield history
521
  return
522
+ elif use_transcription and not audio_url:
523
  # User wants transcription but didn't provide URL
524
  history.append(ChatMessage(
525
  role="assistant",
 
532
  # 4. Claude response phase - for regular chat or when tool isn't needed
533
  try:
534
  # Provide system prompt with tools information
535
+ system_prompt = f"""You are a helpful assistant that can transcribe audio and generate speech from text using specialized tools.
536
 
537
  Available tools:
538
  - Audio transcription: You can convert speech in audio files to text when users provide audio URLs
539
+ - Text-to-Speech: You can convert text to speech audio when users want to generate spoken content
540
 
541
  Tool usage guidelines:
542
+ - The system has already determined that neither the transcription nor TTS tools should be used for this message
543
  - If the user asks about transcribing audio but didn't provide a URL, tell them you need an audio URL
544
+ - If the user asks about generating speech but didn't provide text, tell them you need the text to convert
545
+ - For other queries, respond normally
546
 
547
  When responding to general questions:
548
  - Provide helpful and accurate information
549
  - Be concise and direct
550
+ - If the user's message suggests they might want to use either tool later, you can mention these capabilities
551
  """
552
  # Debug message format
553
  print(f"Sending {len(formatted_messages)} messages to Claude")
 
578
 
579
  # Create the Gradio interface
580
  with gr.Blocks() as demo:
581
+ gr.Markdown("# Claude 3.7 Sonnet Chat with Tools")
582
+ gr.Markdown("Send a message with an audio URL to generate a transcript, or ask to generate speech from text.")
583
+
584
+ # Create a directory for generated audio files if it doesn't exist
585
+ generated_audio_dir = Path("generated_audio")
586
+ generated_audio_dir.mkdir(exist_ok=True)
587
+
588
  with gr.Row():
589
  with gr.Column(scale=4):
590
  chatbot = gr.Chatbot(
591
  type="messages",
592
  label="Claude Agent",
593
+ height=600,
594
+ render=True # Enable HTML rendering for audio player
595
  )
596
  with gr.Column(scale=1):
597
  # Display API key inputs only if environment variables are not set
 
615
 
616
  with gr.Row():
617
  msg = gr.Textbox(
618
+ placeholder="Ask a question, share an audio URL for transcription, or request text-to-speech...",
619
  show_label=False,
620
  container=False,
621
  scale=9