Spaces:
Runtime error
Runtime error
Enrique Cardoza commited on
Commit ·
b116637
1
Parent(s): ce89e4f
feat(tools): add text-to-speech capability to agent
Browse files- Implement TTS API integration with audio/wav response handling
- Create agentic decision-making to identify text-to-speech requests
- Add new function to extract text to convert from user messages
- Implement audio file saving and display in chat interface
- Add proper error handling for TTS generation process
- Update system prompts to include both transcription and TTS capabilities
- Create dedicated directory for storing generated audio files
- Enhance UI messaging and placeholder text to reflect new capability
- Improve chat flow with follow-up responses after audio generation
- Prioritize TTS evaluation before transcription in the decision flow
app.py
CHANGED
|
@@ -2,6 +2,11 @@ import gradio as gr
|
|
| 2 |
import anthropic
|
| 3 |
import os
|
| 4 |
import json
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5 |
from dotenv import load_dotenv
|
| 6 |
from smolagents.mcp_client import MCPClient
|
| 7 |
from gradio import ChatMessage
|
|
@@ -12,6 +17,7 @@ load_dotenv()
|
|
| 12 |
# Get API keys from environment or use None as default
|
| 13 |
anthropic_api_key = os.environ.get("ANTHROPIC_API_KEY")
|
| 14 |
groq_api_key = os.environ.get("GROQ_API_KEY")
|
|
|
|
| 15 |
|
| 16 |
# Initialize Anthropic client (will be updated if user provides a key)
|
| 17 |
client = anthropic.Anthropic(api_key=anthropic_api_key) if anthropic_api_key else None
|
|
@@ -19,6 +25,9 @@ client = anthropic.Anthropic(api_key=anthropic_api_key) if anthropic_api_key els
|
|
| 19 |
# MCP connection details
|
| 20 |
mcp_url = "https://agents-mcp-hackathon-transcript-generator.hf.space/gradio_api/mcp/sse"
|
| 21 |
|
|
|
|
|
|
|
|
|
|
| 22 |
# Global variable to store tools
|
| 23 |
mcp_tools = {}
|
| 24 |
tool_descriptions = {}
|
|
@@ -118,6 +127,44 @@ def process_transcription_from_url(audio_url, user_groq_api_key):
|
|
| 118 |
print(traceback.format_exc())
|
| 119 |
return f"Error: {str(e)}"
|
| 120 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 121 |
# Extract URL from message
|
| 122 |
def extract_url(message):
|
| 123 |
import re
|
|
@@ -128,7 +175,7 @@ def extract_url(message):
|
|
| 128 |
return match.group(0)
|
| 129 |
return None
|
| 130 |
|
| 131 |
-
# Function to decide if tool should be used
|
| 132 |
def should_use_transcription_tool(prompt, available_tools):
|
| 133 |
try:
|
| 134 |
if not client:
|
|
@@ -191,6 +238,69 @@ def should_use_transcription_tool(prompt, available_tools):
|
|
| 191 |
# Default to not using tool if there's an error
|
| 192 |
return False, f"Error: {str(e)}"
|
| 193 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 194 |
# Main chat function (streaming, with tool usage)
|
| 195 |
def chat_with_tools(user_message, history, user_groq_api_key, user_anthropic_api_key):
|
| 196 |
global client, anthropic_api_key, groq_api_key, tool_descriptions
|
|
@@ -220,20 +330,110 @@ def chat_with_tools(user_message, history, user_groq_api_key, user_anthropic_api
|
|
| 220 |
# 1. "Thinking" phase
|
| 221 |
yield history + [ChatMessage(role="assistant", content="Let me think...", metadata={"title": "🧠 Thinking", "status": "pending"})]
|
| 222 |
|
| 223 |
-
# Extract URL from message
|
| 224 |
audio_url = extract_url(user_message)
|
| 225 |
|
| 226 |
-
# 2. Decision phase -
|
| 227 |
-
|
| 228 |
-
reasoning = ""
|
| 229 |
|
| 230 |
-
if
|
| 231 |
-
|
| 232 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 233 |
|
| 234 |
-
#
|
| 235 |
-
|
| 236 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 237 |
try:
|
| 238 |
# Check if GROQ API key is available (either from env or user input)
|
| 239 |
current_groq_api_key = groq_api_key if groq_api_key else user_groq_api_key
|
|
@@ -319,7 +519,7 @@ def chat_with_tools(user_message, history, user_groq_api_key, user_anthropic_api
|
|
| 319 |
))
|
| 320 |
yield history
|
| 321 |
return
|
| 322 |
-
elif
|
| 323 |
# User wants transcription but didn't provide URL
|
| 324 |
history.append(ChatMessage(
|
| 325 |
role="assistant",
|
|
@@ -332,20 +532,22 @@ def chat_with_tools(user_message, history, user_groq_api_key, user_anthropic_api
|
|
| 332 |
# 4. Claude response phase - for regular chat or when tool isn't needed
|
| 333 |
try:
|
| 334 |
# Provide system prompt with tools information
|
| 335 |
-
system_prompt = f"""You are a helpful assistant that can transcribe audio using specialized tools.
|
| 336 |
|
| 337 |
Available tools:
|
| 338 |
- Audio transcription: You can convert speech in audio files to text when users provide audio URLs
|
|
|
|
| 339 |
|
| 340 |
Tool usage guidelines:
|
| 341 |
-
- The system has already determined that the transcription
|
| 342 |
- If the user asks about transcribing audio but didn't provide a URL, tell them you need an audio URL
|
| 343 |
-
-
|
|
|
|
| 344 |
|
| 345 |
When responding to general questions:
|
| 346 |
- Provide helpful and accurate information
|
| 347 |
- Be concise and direct
|
| 348 |
-
- If the user's message suggests they might want to
|
| 349 |
"""
|
| 350 |
# Debug message format
|
| 351 |
print(f"Sending {len(formatted_messages)} messages to Claude")
|
|
@@ -376,14 +578,20 @@ initialize_tools()
|
|
| 376 |
|
| 377 |
# Create the Gradio interface
|
| 378 |
with gr.Blocks() as demo:
|
| 379 |
-
gr.Markdown("# Claude 3.7 Sonnet Chat with
|
| 380 |
-
gr.Markdown("Send a message with an audio URL to generate a transcript
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 381 |
with gr.Row():
|
| 382 |
with gr.Column(scale=4):
|
| 383 |
chatbot = gr.Chatbot(
|
| 384 |
type="messages",
|
| 385 |
label="Claude Agent",
|
| 386 |
-
height=600
|
|
|
|
| 387 |
)
|
| 388 |
with gr.Column(scale=1):
|
| 389 |
# Display API key inputs only if environment variables are not set
|
|
@@ -407,7 +615,7 @@ with gr.Blocks() as demo:
|
|
| 407 |
|
| 408 |
with gr.Row():
|
| 409 |
msg = gr.Textbox(
|
| 410 |
-
placeholder="Ask a question
|
| 411 |
show_label=False,
|
| 412 |
container=False,
|
| 413 |
scale=9
|
|
|
|
| 2 |
import anthropic
|
| 3 |
import os
|
| 4 |
import json
|
| 5 |
+
import requests
|
| 6 |
+
import base64
|
| 7 |
+
import tempfile
|
| 8 |
+
import time
|
| 9 |
+
from pathlib import Path
|
| 10 |
from dotenv import load_dotenv
|
| 11 |
from smolagents.mcp_client import MCPClient
|
| 12 |
from gradio import ChatMessage
|
|
|
|
| 17 |
# Get API keys from environment or use None as default
|
| 18 |
anthropic_api_key = os.environ.get("ANTHROPIC_API_KEY")
|
| 19 |
groq_api_key = os.environ.get("GROQ_API_KEY")
|
| 20 |
+
tts_api_url_base = os.environ.get("TTS_API_URL")
|
| 21 |
|
| 22 |
# Initialize Anthropic client (will be updated if user provides a key)
|
| 23 |
client = anthropic.Anthropic(api_key=anthropic_api_key) if anthropic_api_key else None
|
|
|
|
| 25 |
# MCP connection details
|
| 26 |
mcp_url = "https://agents-mcp-hackathon-transcript-generator.hf.space/gradio_api/mcp/sse"
|
| 27 |
|
| 28 |
+
# TTS API endpoint
|
| 29 |
+
tts_api_url = tts_api_url_base+"/generate"
|
| 30 |
+
|
| 31 |
# Global variable to store tools
|
| 32 |
mcp_tools = {}
|
| 33 |
tool_descriptions = {}
|
|
|
|
| 127 |
print(traceback.format_exc())
|
| 128 |
return f"Error: {str(e)}"
|
| 129 |
|
| 130 |
+
# Generate speech from text using the TTS API
|
| 131 |
+
def generate_speech_from_text(text):
|
| 132 |
+
try:
|
| 133 |
+
# Prepare the request data
|
| 134 |
+
payload = {"prompt": text}
|
| 135 |
+
headers = {"Content-Type": "application/json"}
|
| 136 |
+
|
| 137 |
+
print(f"Sending TTS request for text (length): {len(text)}")
|
| 138 |
+
|
| 139 |
+
# Make the POST request to the TTS API
|
| 140 |
+
response = requests.post(tts_api_url, json=payload, headers=headers)
|
| 141 |
+
|
| 142 |
+
# Check if the request was successful
|
| 143 |
+
if response.status_code == 200:
|
| 144 |
+
print(f"TTS response received. Content-Type: {response.headers.get('Content-Type')}")
|
| 145 |
+
|
| 146 |
+
# Ensure the output directory exists
|
| 147 |
+
output_dir = Path("generated_audio")
|
| 148 |
+
output_dir.mkdir(exist_ok=True)
|
| 149 |
+
|
| 150 |
+
# Create a file to store the audio with a timestamp to avoid conflicts
|
| 151 |
+
timestamp = int(time.time())
|
| 152 |
+
audio_file_path = output_dir / f"speech_{timestamp}.wav"
|
| 153 |
+
|
| 154 |
+
# Write the audio bytes to the file
|
| 155 |
+
with open(audio_file_path, "wb") as f:
|
| 156 |
+
f.write(response.content)
|
| 157 |
+
|
| 158 |
+
print(f"Audio saved to file: {audio_file_path}")
|
| 159 |
+
return str(audio_file_path)
|
| 160 |
+
else:
|
| 161 |
+
return f"Error: API returned status code {response.status_code} - {response.text}"
|
| 162 |
+
except Exception as e:
|
| 163 |
+
import traceback
|
| 164 |
+
print(f"TTS error: {e}")
|
| 165 |
+
print(traceback.format_exc())
|
| 166 |
+
return f"Error: {str(e)}"
|
| 167 |
+
|
| 168 |
# Extract URL from message
|
| 169 |
def extract_url(message):
|
| 170 |
import re
|
|
|
|
| 175 |
return match.group(0)
|
| 176 |
return None
|
| 177 |
|
| 178 |
+
# Function to decide if transcription tool should be used
|
| 179 |
def should_use_transcription_tool(prompt, available_tools):
|
| 180 |
try:
|
| 181 |
if not client:
|
|
|
|
| 238 |
# Default to not using tool if there's an error
|
| 239 |
return False, f"Error: {str(e)}"
|
| 240 |
|
| 241 |
+
# Function to decide if TTS tool should be used
|
| 242 |
+
def should_use_tts_tool(prompt):
|
| 243 |
+
try:
|
| 244 |
+
if not client:
|
| 245 |
+
print("No Claude client available for TTS tool decision")
|
| 246 |
+
# Default to not using tool since we can't ask Claude
|
| 247 |
+
return False, "No Claude client available for decision", ""
|
| 248 |
+
|
| 249 |
+
system_prompt = """You are an AI assistant deciding whether to use a text-to-speech (TTS) tool.
|
| 250 |
+
You have access to a TTS tool that can generate speech audio from text.
|
| 251 |
+
|
| 252 |
+
You should ONLY respond with a pure JSON object that includes:
|
| 253 |
+
1. "use_tool": true/false - whether to use the TTS tool
|
| 254 |
+
2. "reasoning": brief explanation of your decision
|
| 255 |
+
3. "text_to_convert": If use_tool is true, include the text that should be converted to speech.
|
| 256 |
+
Extract this from the user's message or generate appropriate text based on their request.
|
| 257 |
+
|
| 258 |
+
It should start with a '{' and end with a '}'.
|
| 259 |
+
|
| 260 |
+
Respond with true ONLY if:
|
| 261 |
+
- The user is clearly asking to generate speech or audio from text
|
| 262 |
+
- OR the user wants text read aloud
|
| 263 |
+
- OR the user wants to create a voiceover or narration
|
| 264 |
+
- OR the user explicitly mentions text-to-speech or TTS
|
| 265 |
+
|
| 266 |
+
When deciding what text to convert:
|
| 267 |
+
- If the user provides specific text in quotes or after phrases like "convert this to speech:", use that exact text
|
| 268 |
+
- If the user asks to convert their previous message, use that as the text
|
| 269 |
+
- If the user doesn't specify text but clearly wants TTS, ask them to provide the text
|
| 270 |
+
|
| 271 |
+
Respond with false if:
|
| 272 |
+
- The user is just chatting or asking questions not related to speech generation
|
| 273 |
+
- The user is asking about transcription (speech-to-text) instead
|
| 274 |
+
"""
|
| 275 |
+
|
| 276 |
+
# Ensure we're sending a valid message
|
| 277 |
+
response = client.messages.create(
|
| 278 |
+
model="claude-3-7-sonnet-latest",
|
| 279 |
+
max_tokens=500,
|
| 280 |
+
system=system_prompt,
|
| 281 |
+
messages=[{"role": "user", "content": f"User message: {prompt}"}]
|
| 282 |
+
)
|
| 283 |
+
|
| 284 |
+
response_content = response.content[0].text
|
| 285 |
+
print(f"TTS tool decision response: {response_content}")
|
| 286 |
+
|
| 287 |
+
# Try to parse the JSON response
|
| 288 |
+
try:
|
| 289 |
+
result = json.loads(response_content)
|
| 290 |
+
text_to_convert = result.get("text_to_convert", "")
|
| 291 |
+
return result.get("use_tool", False), result.get("reasoning", "No reasoning provided"), text_to_convert
|
| 292 |
+
except json.JSONDecodeError:
|
| 293 |
+
print(f"Failed to parse JSON response: {response_content}")
|
| 294 |
+
# Fallback: check if the response contains "true"
|
| 295 |
+
return "true" in response_content.lower(), "Failed to parse JSON response", ""
|
| 296 |
+
|
| 297 |
+
except Exception as e:
|
| 298 |
+
import traceback
|
| 299 |
+
print(f"Error in TTS tool decision: {e}")
|
| 300 |
+
print(traceback.format_exc())
|
| 301 |
+
# Default to not using tool if there's an error
|
| 302 |
+
return False, f"Error: {str(e)}", ""
|
| 303 |
+
|
| 304 |
# Main chat function (streaming, with tool usage)
|
| 305 |
def chat_with_tools(user_message, history, user_groq_api_key, user_anthropic_api_key):
|
| 306 |
global client, anthropic_api_key, groq_api_key, tool_descriptions
|
|
|
|
| 330 |
# 1. "Thinking" phase
|
| 331 |
yield history + [ChatMessage(role="assistant", content="Let me think...", metadata={"title": "🧠 Thinking", "status": "pending"})]
|
| 332 |
|
| 333 |
+
# Extract URL from message for transcription
|
| 334 |
audio_url = extract_url(user_message)
|
| 335 |
|
| 336 |
+
# 2. Decision phase - First check if we should use the TTS tool
|
| 337 |
+
use_tts, tts_reasoning, text_to_convert = should_use_tts_tool(user_message)
|
|
|
|
| 338 |
|
| 339 |
+
# 3a. Tool usage phase - Process TTS if decided to use that tool
|
| 340 |
+
if use_tts:
|
| 341 |
+
try:
|
| 342 |
+
# Show tool call status
|
| 343 |
+
yield history + [ChatMessage(
|
| 344 |
+
role="assistant",
|
| 345 |
+
content=f"Generating speech from text...",
|
| 346 |
+
metadata={"title": "🔊 Text-to-Speech", "status": "pending"}
|
| 347 |
+
)]
|
| 348 |
+
|
| 349 |
+
# If no text was provided for conversion, ask the user
|
| 350 |
+
if not text_to_convert:
|
| 351 |
+
history.append(ChatMessage(
|
| 352 |
+
role="assistant",
|
| 353 |
+
content="I'd be happy to generate speech audio for you! Could you please provide the text you want me to convert to speech?",
|
| 354 |
+
metadata={"title": "ℹ️ Info", "status": "done"}
|
| 355 |
+
))
|
| 356 |
+
yield history
|
| 357 |
+
return
|
| 358 |
+
|
| 359 |
+
# Process text-to-speech
|
| 360 |
+
audio_file_path = generate_speech_from_text(text_to_convert)
|
| 361 |
+
|
| 362 |
+
if audio_file_path and isinstance(audio_file_path, str) and audio_file_path.startswith("Error:"):
|
| 363 |
+
history.append(ChatMessage(
|
| 364 |
+
role="assistant",
|
| 365 |
+
content=audio_file_path,
|
| 366 |
+
metadata={"title": "❌ Error", "status": "done"}
|
| 367 |
+
))
|
| 368 |
+
yield history
|
| 369 |
+
return
|
| 370 |
+
elif audio_file_path:
|
| 371 |
+
# Add the audio component message
|
| 372 |
+
audio_display_message = f"""I've generated speech audio from your text. You can play it below:
|
| 373 |
+
|
| 374 |
+
<audio controls src="/file={audio_file_path}" style="width: 100%;"></audio>
|
| 375 |
+
|
| 376 |
+
The text that was converted: "{text_to_convert}"
|
| 377 |
+
"""
|
| 378 |
+
history.append(ChatMessage(
|
| 379 |
+
role="assistant",
|
| 380 |
+
content=audio_display_message,
|
| 381 |
+
metadata={"title": "🔊 Speech Generated", "status": "done"}
|
| 382 |
+
))
|
| 383 |
+
|
| 384 |
+
yield history
|
| 385 |
+
|
| 386 |
+
# Let Claude provide a response about the generated audio
|
| 387 |
+
try:
|
| 388 |
+
# Create a prompt for Claude to respond about the TTS generation
|
| 389 |
+
claude_prompt = f"You've just generated speech audio for the user from this text: \"{text_to_convert}\". Please provide a brief, helpful response. You might ask if the audio meets their needs or if they'd like to convert more text."
|
| 390 |
+
|
| 391 |
+
# Send request to Claude
|
| 392 |
+
response = client.messages.create(
|
| 393 |
+
model="claude-3-7-sonnet-latest",
|
| 394 |
+
max_tokens=1024,
|
| 395 |
+
system="You are a helpful assistant that can convert text to speech using specialized tools. When responding to the user about generated speech, be brief and helpful. Don't repeat the full text that was converted unless it's very short.",
|
| 396 |
+
messages=[{"role": "user", "content": claude_prompt}]
|
| 397 |
+
)
|
| 398 |
+
|
| 399 |
+
# Add Claude's response to history
|
| 400 |
+
response_content = response.content[0].text
|
| 401 |
+
history.append(ChatMessage(role="assistant", content=response_content))
|
| 402 |
+
yield history
|
| 403 |
+
return
|
| 404 |
+
except Exception as e:
|
| 405 |
+
print(f"Error generating response about TTS: {str(e)}")
|
| 406 |
+
# Continue without Claude's additional response
|
| 407 |
+
else:
|
| 408 |
+
history.append(ChatMessage(
|
| 409 |
+
role="assistant",
|
| 410 |
+
content="I wasn't able to generate speech from your text. Please try again with different text.",
|
| 411 |
+
metadata={"title": "❌ Error", "status": "done"}
|
| 412 |
+
))
|
| 413 |
+
yield history
|
| 414 |
+
return
|
| 415 |
+
except Exception as e:
|
| 416 |
+
import traceback
|
| 417 |
+
error_details = traceback.format_exc()
|
| 418 |
+
print(f"TTS error details: {error_details}")
|
| 419 |
+
|
| 420 |
+
history.append(ChatMessage(
|
| 421 |
+
role="assistant",
|
| 422 |
+
content=f"Error generating speech: {str(e)}",
|
| 423 |
+
metadata={"title": "❌ Error", "status": "done"}
|
| 424 |
+
))
|
| 425 |
+
yield history
|
| 426 |
+
return
|
| 427 |
|
| 428 |
+
# 2b. If not using TTS, check if we should use the transcription tool
|
| 429 |
+
if not use_tts and audio_url:
|
| 430 |
+
use_transcription, transcription_reasoning = should_use_transcription_tool(user_message, tool_descriptions)
|
| 431 |
+
print(f"Tool decision: {use_transcription}, Reasoning: {transcription_reasoning}")
|
| 432 |
+
else:
|
| 433 |
+
use_transcription = False
|
| 434 |
+
|
| 435 |
+
# 3b. Process transcription if decided to use that tool
|
| 436 |
+
if use_transcription and audio_url:
|
| 437 |
try:
|
| 438 |
# Check if GROQ API key is available (either from env or user input)
|
| 439 |
current_groq_api_key = groq_api_key if groq_api_key else user_groq_api_key
|
|
|
|
| 519 |
))
|
| 520 |
yield history
|
| 521 |
return
|
| 522 |
+
elif use_transcription and not audio_url:
|
| 523 |
# User wants transcription but didn't provide URL
|
| 524 |
history.append(ChatMessage(
|
| 525 |
role="assistant",
|
|
|
|
| 532 |
# 4. Claude response phase - for regular chat or when tool isn't needed
|
| 533 |
try:
|
| 534 |
# Provide system prompt with tools information
|
| 535 |
+
system_prompt = f"""You are a helpful assistant that can transcribe audio and generate speech from text using specialized tools.
|
| 536 |
|
| 537 |
Available tools:
|
| 538 |
- Audio transcription: You can convert speech in audio files to text when users provide audio URLs
|
| 539 |
+
- Text-to-Speech: You can convert text to speech audio when users want to generate spoken content
|
| 540 |
|
| 541 |
Tool usage guidelines:
|
| 542 |
+
- The system has already determined that neither the transcription nor TTS tools should be used for this message
|
| 543 |
- If the user asks about transcribing audio but didn't provide a URL, tell them you need an audio URL
|
| 544 |
+
- If the user asks about generating speech but didn't provide text, tell them you need the text to convert
|
| 545 |
+
- For other queries, respond normally
|
| 546 |
|
| 547 |
When responding to general questions:
|
| 548 |
- Provide helpful and accurate information
|
| 549 |
- Be concise and direct
|
| 550 |
+
- If the user's message suggests they might want to use either tool later, you can mention these capabilities
|
| 551 |
"""
|
| 552 |
# Debug message format
|
| 553 |
print(f"Sending {len(formatted_messages)} messages to Claude")
|
|
|
|
| 578 |
|
| 579 |
# Create the Gradio interface
|
| 580 |
with gr.Blocks() as demo:
|
| 581 |
+
gr.Markdown("# Claude 3.7 Sonnet Chat with Tools")
|
| 582 |
+
gr.Markdown("Send a message with an audio URL to generate a transcript, or ask to generate speech from text.")
|
| 583 |
+
|
| 584 |
+
# Create a directory for generated audio files if it doesn't exist
|
| 585 |
+
generated_audio_dir = Path("generated_audio")
|
| 586 |
+
generated_audio_dir.mkdir(exist_ok=True)
|
| 587 |
+
|
| 588 |
with gr.Row():
|
| 589 |
with gr.Column(scale=4):
|
| 590 |
chatbot = gr.Chatbot(
|
| 591 |
type="messages",
|
| 592 |
label="Claude Agent",
|
| 593 |
+
height=600,
|
| 594 |
+
render=True # Enable HTML rendering for audio player
|
| 595 |
)
|
| 596 |
with gr.Column(scale=1):
|
| 597 |
# Display API key inputs only if environment variables are not set
|
|
|
|
| 615 |
|
| 616 |
with gr.Row():
|
| 617 |
msg = gr.Textbox(
|
| 618 |
+
placeholder="Ask a question, share an audio URL for transcription, or request text-to-speech...",
|
| 619 |
show_label=False,
|
| 620 |
container=False,
|
| 621 |
scale=9
|