Devakumar868 commited on
Commit
79a976b
Β·
verified Β·
1 Parent(s): 3ab8fbe

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +211 -102
app.py CHANGED
@@ -2,108 +2,169 @@ import gradio as gr
2
  import torch
3
  import numpy as np
4
  import librosa
5
- from transformers import pipeline, AutoTokenizer, AutoModelForCausalLM
6
  import soundfile as sf
7
- from huggingface_hub import hf_hub_download
8
  import json
9
  import time
10
  from datetime import datetime
11
  import os
 
 
 
 
12
 
13
- # Initialize models
14
  class ConversationalAI:
15
  def __init__(self):
16
- # Load ASR model (using Whisper as fallback since Parakeet may not be available)
17
- self.asr_model = pipeline("automatic-speech-recognition",
18
- model="openai/whisper-large-v3",
19
- torch_dtype=torch.float16,
20
- device="cuda" if torch.cuda.is_available() else "cpu")
 
 
 
 
 
21
 
22
- # Load LLM (using smaller model for HF Spaces)
23
  self.llm_tokenizer = AutoTokenizer.from_pretrained("microsoft/DialoGPT-medium")
 
24
  self.llm_model = AutoModelForCausalLM.from_pretrained(
25
  "microsoft/DialoGPT-medium",
26
- torch_dtype=torch.float16,
27
- device_map="auto"
28
- )
29
 
30
  # Load TTS model
31
- self.tts_model = pipeline("text-to-speech",
32
- model="microsoft/speecht5_tts",
33
- torch_dtype=torch.float16,
34
- device="cuda" if torch.cuda.is_available() else "cpu")
 
 
35
 
36
- # Load emotion recognition
37
- self.emotion_model = pipeline("audio-classification",
38
- model="ehcalabres/wav2vec2-lg-xlsr-en-speech-emotion-recognition",
39
- device="cuda" if torch.cuda.is_available() else "cpu")
 
 
40
 
41
  # Conversation history
42
  self.conversations = {}
43
 
44
  def transcribe_audio(self, audio_path):
45
- """Transcribe audio using Whisper"""
46
  try:
47
  if audio_path is None:
48
  return "No audio provided"
49
 
50
- result = self.asr_model(audio_path)
51
- return result["text"]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
52
  except Exception as e:
53
  return f"Transcription error: {str(e)}"
54
 
55
- def recognize_emotion(self, audio_path):
56
- """Recognize emotion from audio"""
57
  try:
58
- if audio_path is None:
59
  return "neutral"
60
 
61
- result = self.emotion_model(audio_path)
62
  return result[0]["label"].lower()
63
  except:
64
  return "neutral"
65
 
66
  def generate_response(self, text, emotion, conversation_history):
67
- """Generate contextual response"""
68
  try:
69
- # Build context-aware prompt
70
- context = f"Previous conversation: {conversation_history[-2:] if conversation_history else 'None'}"
71
- emotion_context = f"User emotion: {emotion}"
72
 
73
- prompt = f"You are Maya, a friendly AI assistant. {context} {emotion_context} User: {text} Maya:"
 
 
74
 
75
- inputs = self.llm_tokenizer.encode(prompt, return_tensors="pt")
 
 
 
 
 
 
 
76
 
77
  with torch.no_grad():
78
  outputs = self.llm_model.generate(
79
- inputs,
80
- max_new_tokens=100,
 
81
  temperature=0.7,
82
  do_sample=True,
83
- pad_token_id=self.llm_tokenizer.eos_token_id
 
84
  )
85
 
86
- response = self.llm_tokenizer.decode(outputs[0], skip_special_tokens=True)
87
- # Extract only the new response
88
- response = response.split("Maya:")[-1].strip()
 
 
 
 
 
 
89
 
90
  return response
 
91
  except Exception as e:
92
- return f"I'm sorry, I encountered an error: {str(e)}"
93
 
94
  def synthesize_speech(self, text):
95
  """Generate speech using TTS"""
96
  try:
97
- # Use a simple TTS approach for HF Spaces
98
- audio = self.tts_model(text)
 
 
 
 
 
 
 
99
  return audio["audio"]
 
100
  except Exception as e:
 
101
  return None
102
 
103
  def process_conversation(self, audio_input, user_id="default"):
104
  """Main conversation processing pipeline"""
105
  if audio_input is None:
106
- return "Please provide audio input", None, "No conversation yet"
107
 
108
  start_time = time.time()
109
 
@@ -111,50 +172,58 @@ class ConversationalAI:
111
  if user_id not in self.conversations:
112
  self.conversations[user_id] = []
113
 
114
- # Step 1: Transcribe audio
115
- transcription = self.transcribe_audio(audio_input)
116
-
117
- # Step 2: Recognize emotion
118
- emotion = self.recognize_emotion(audio_input)
119
-
120
- # Step 3: Generate response
121
- response_text = self.generate_response(
122
- transcription, emotion, self.conversations[user_id]
123
- )
124
-
125
- # Step 4: Synthesize speech
126
- response_audio = self.synthesize_speech(response_text)
127
-
128
- # Step 5: Update conversation history
129
- conversation_entry = {
130
- "timestamp": datetime.now().isoformat(),
131
- "user_input": transcription,
132
- "user_emotion": emotion,
133
- "ai_response": response_text,
134
- "processing_time": time.time() - start_time
135
- }
136
-
137
- self.conversations[user_id].append(conversation_entry)
138
-
139
- # Keep only last 20 exchanges per user
140
- if len(self.conversations[user_id]) > 20:
141
- self.conversations[user_id] = self.conversations[user_id][-20:]
142
-
143
- # Format conversation history
144
- history = self.format_conversation_history(user_id)
145
-
146
- return transcription, response_audio, history
 
 
 
 
 
 
147
 
148
  def format_conversation_history(self, user_id):
149
  """Format conversation history for display"""
150
- if user_id not in self.conversations:
151
- return "No conversation history"
152
 
153
  history = []
154
- for entry in self.conversations[user_id][-5:]: # Show last 5 exchanges
155
- history.append(f"🎀 You ({entry['user_emotion']}): {entry['user_input']}")
156
- history.append(f"πŸ€– Maya: {entry['ai_response']}")
157
- history.append(f"⏱️ Response time: {entry['processing_time']:.2f}s\n")
 
 
158
 
159
  return "\n".join(history)
160
 
@@ -162,55 +231,83 @@ class ConversationalAI:
162
  """Clear conversation history"""
163
  if user_id in self.conversations:
164
  self.conversations[user_id] = []
165
- return "Conversation cleared!"
166
 
167
  # Initialize the AI system
 
168
  ai_system = ConversationalAI()
 
169
 
170
  # Gradio interface functions
171
  def process_audio(audio):
172
  if audio is None:
173
- return "No audio provided", None, "No conversation yet"
174
 
175
- transcription, response_audio, history = ai_system.process_conversation(audio)
176
- return transcription, response_audio, history
177
 
178
  def clear_chat():
179
  message = ai_system.clear_conversation()
180
- return "", "Conversation cleared!"
 
 
 
181
 
182
  # Create Gradio interface
183
- with gr.Blocks(title="Maya AI - Conversational Assistant", theme=gr.themes.Soft()) as demo:
184
- gr.Markdown("# 🎀 Maya AI - Your Conversational Partner")
185
- gr.Markdown("*Speak naturally and Maya will respond with voice and emotion recognition*")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
186
 
187
  with gr.Row():
188
  with gr.Column(scale=1):
 
189
  audio_input = gr.Audio(
190
  sources=["microphone"],
191
  type="filepath",
192
- label="πŸŽ™οΈ Speak to Maya"
 
193
  )
194
 
195
- process_btn = gr.Button("πŸ’¬ Process", variant="primary")
196
- clear_btn = gr.Button("πŸ—‘οΈ Clear Chat", variant="secondary")
 
197
 
198
  with gr.Column(scale=2):
 
199
  transcription_output = gr.Textbox(
200
- label="πŸ“ What you said",
201
  lines=2,
202
- interactive=False
 
203
  )
204
 
205
  audio_output = gr.Audio(
206
  label="πŸ”Š Maya's Response",
207
- interactive=False
 
208
  )
209
 
210
  conversation_history = gr.Textbox(
211
  label="πŸ’­ Conversation History",
212
- lines=10,
213
- interactive=False
 
 
214
  )
215
 
216
  # Event handlers
@@ -222,16 +319,28 @@ with gr.Blocks(title="Maya AI - Conversational Assistant", theme=gr.themes.Soft(
222
 
223
  clear_btn.click(
224
  fn=clear_chat,
225
- outputs=[transcription_output, conversation_history]
226
  )
227
 
228
- # Auto-process when audio is uploaded
229
- audio_input.change(
230
  fn=process_audio,
231
  inputs=[audio_input],
232
  outputs=[transcription_output, audio_output, conversation_history]
233
  )
 
 
 
 
 
 
234
 
235
  # Launch the app
236
  if __name__ == "__main__":
237
- demo.launch()
 
 
 
 
 
 
 
2
  import torch
3
  import numpy as np
4
  import librosa
5
+ from transformers import pipeline, AutoTokenizer, AutoModelForCausalLM, WhisperProcessor, WhisperForConditionalGeneration
6
  import soundfile as sf
 
7
  import json
8
  import time
9
  from datetime import datetime
10
  import os
11
+ import warnings
12
+
13
+ # Suppress warnings for cleaner output
14
+ warnings.filterwarnings("ignore")
15
 
 
16
  class ConversationalAI:
17
  def __init__(self):
18
+ # Set device
19
+ self.device = "cuda" if torch.cuda.is_available() else "cpu"
20
+ print(f"Using device: {self.device}")
21
+
22
+ # Load Whisper ASR with proper configuration
23
+ self.asr_processor = WhisperProcessor.from_pretrained("openai/whisper-base.en")
24
+ self.asr_model = WhisperForConditionalGeneration.from_pretrained(
25
+ "openai/whisper-base.en",
26
+ torch_dtype=torch.float16 if self.device == "cuda" else torch.float32
27
+ ).to(self.device)
28
 
29
+ # Load LLM with proper device handling
30
  self.llm_tokenizer = AutoTokenizer.from_pretrained("microsoft/DialoGPT-medium")
31
+ self.llm_tokenizer.pad_token = self.llm_tokenizer.eos_token
32
  self.llm_model = AutoModelForCausalLM.from_pretrained(
33
  "microsoft/DialoGPT-medium",
34
+ torch_dtype=torch.float16 if self.device == "cuda" else torch.float32,
35
+ pad_token_id=self.llm_tokenizer.eos_token_id
36
+ ).to(self.device)
37
 
38
  # Load TTS model
39
+ self.tts_model = pipeline(
40
+ "text-to-speech",
41
+ model="microsoft/speecht5_tts",
42
+ torch_dtype=torch.float16 if self.device == "cuda" else torch.float32,
43
+ device=self.device
44
+ )
45
 
46
+ # Load emotion recognition with better model
47
+ self.emotion_model = pipeline(
48
+ "audio-classification",
49
+ model="j-hartmann/emotion-english-distilroberta-base",
50
+ device=self.device
51
+ )
52
 
53
  # Conversation history
54
  self.conversations = {}
55
 
56
  def transcribe_audio(self, audio_path):
57
+ """Transcribe audio using Whisper with proper device handling"""
58
  try:
59
  if audio_path is None:
60
  return "No audio provided"
61
 
62
+ # Load and preprocess audio
63
+ audio, sr = librosa.load(audio_path, sr=16000, mono=True)
64
+
65
+ # Process with Whisper
66
+ inputs = self.asr_processor(
67
+ audio,
68
+ sampling_rate=16000,
69
+ return_tensors="pt",
70
+ language="en"
71
+ ).to(self.device)
72
+
73
+ with torch.no_grad():
74
+ predicted_ids = self.asr_model.generate(
75
+ inputs.input_features,
76
+ max_new_tokens=100,
77
+ do_sample=False
78
+ )
79
+
80
+ transcription = self.asr_processor.batch_decode(
81
+ predicted_ids,
82
+ skip_special_tokens=True
83
+ )[0]
84
+
85
+ return transcription.strip()
86
+
87
  except Exception as e:
88
  return f"Transcription error: {str(e)}"
89
 
90
+ def recognize_emotion(self, text):
91
+ """Recognize emotion from text (more reliable than audio emotion)"""
92
  try:
93
+ if not text or text.startswith("Transcription error"):
94
  return "neutral"
95
 
96
+ result = self.emotion_model(text)
97
  return result[0]["label"].lower()
98
  except:
99
  return "neutral"
100
 
101
  def generate_response(self, text, emotion, conversation_history):
102
+ """Generate contextual response with proper device handling"""
103
  try:
104
+ if text.startswith("Transcription error") or not text.strip():
105
+ return "I'm sorry, I couldn't understand what you said. Could you please try again?"
 
106
 
107
+ # Build context-aware prompt
108
+ emotion_prompt = f"[User seems {emotion}] " if emotion != "neutral" else ""
109
+ prompt = f"{emotion_prompt}User: {text}\nMaya:"
110
 
111
+ # Tokenize with proper attention mask
112
+ inputs = self.llm_tokenizer(
113
+ prompt,
114
+ return_tensors="pt",
115
+ padding=True,
116
+ truncation=True,
117
+ max_length=512
118
+ ).to(self.device)
119
 
120
  with torch.no_grad():
121
  outputs = self.llm_model.generate(
122
+ input_ids=inputs.input_ids,
123
+ attention_mask=inputs.attention_mask,
124
+ max_new_tokens=80,
125
  temperature=0.7,
126
  do_sample=True,
127
+ pad_token_id=self.llm_tokenizer.eos_token_id,
128
+ eos_token_id=self.llm_tokenizer.eos_token_id
129
  )
130
 
131
+ # Decode response
132
+ response = self.llm_tokenizer.decode(
133
+ outputs[0][inputs.input_ids.shape[1]:],
134
+ skip_special_tokens=True
135
+ ).strip()
136
+
137
+ # Clean up response
138
+ if not response:
139
+ response = "I understand. Could you tell me more about that?"
140
 
141
  return response
142
+
143
  except Exception as e:
144
+ return "I'm here to help. What would you like to talk about?"
145
 
146
  def synthesize_speech(self, text):
147
  """Generate speech using TTS"""
148
  try:
149
+ if not text or len(text.strip()) == 0:
150
+ return None
151
+
152
+ # Clean text for TTS
153
+ clean_text = text.replace("[", "").replace("]", "").strip()
154
+ if len(clean_text) > 200:
155
+ clean_text = clean_text[:200] + "..."
156
+
157
+ audio = self.tts_model(clean_text)
158
  return audio["audio"]
159
+
160
  except Exception as e:
161
+ print(f"TTS error: {e}")
162
  return None
163
 
164
  def process_conversation(self, audio_input, user_id="default"):
165
  """Main conversation processing pipeline"""
166
  if audio_input is None:
167
+ return "Please record some audio first", None, "No conversation yet"
168
 
169
  start_time = time.time()
170
 
 
172
  if user_id not in self.conversations:
173
  self.conversations[user_id] = []
174
 
175
+ try:
176
+ # Step 1: Transcribe audio
177
+ transcription = self.transcribe_audio(audio_input)
178
+
179
+ # Step 2: Recognize emotion from text
180
+ emotion = self.recognize_emotion(transcription)
181
+
182
+ # Step 3: Generate response
183
+ response_text = self.generate_response(
184
+ transcription, emotion, self.conversations[user_id]
185
+ )
186
+
187
+ # Step 4: Synthesize speech
188
+ response_audio = self.synthesize_speech(response_text)
189
+
190
+ # Step 5: Update conversation history
191
+ processing_time = time.time() - start_time
192
+ conversation_entry = {
193
+ "timestamp": datetime.now().strftime("%H:%M:%S"),
194
+ "user_input": transcription,
195
+ "user_emotion": emotion,
196
+ "ai_response": response_text,
197
+ "processing_time": processing_time
198
+ }
199
+
200
+ self.conversations[user_id].append(conversation_entry)
201
+
202
+ # Keep only last 15 exchanges per user
203
+ if len(self.conversations[user_id]) > 15:
204
+ self.conversations[user_id] = self.conversations[user_id][-15:]
205
+
206
+ # Format conversation history
207
+ history = self.format_conversation_history(user_id)
208
+
209
+ return transcription, response_audio, history
210
+
211
+ except Exception as e:
212
+ error_msg = f"Processing error: {str(e)}"
213
+ return error_msg, None, "Error occurred during processing"
214
 
215
  def format_conversation_history(self, user_id):
216
  """Format conversation history for display"""
217
+ if user_id not in self.conversations or not self.conversations[user_id]:
218
+ return "No conversation history yet. Start by recording some audio!"
219
 
220
  history = []
221
+ for i, entry in enumerate(self.conversations[user_id][-5:], 1):
222
+ history.append(f"**Exchange {i}** ({entry['timestamp']})")
223
+ history.append(f"🎀 **You** ({entry['user_emotion']}): {entry['user_input']}")
224
+ history.append(f"πŸ€– **Maya**: {entry['ai_response']}")
225
+ history.append(f"⏱️ *Response time: {entry['processing_time']:.2f}s*")
226
+ history.append("---")
227
 
228
  return "\n".join(history)
229
 
 
231
  """Clear conversation history"""
232
  if user_id in self.conversations:
233
  self.conversations[user_id] = []
234
+ return "Conversation cleared! Ready for a fresh start."
235
 
236
  # Initialize the AI system
237
+ print("Initializing Maya AI...")
238
  ai_system = ConversationalAI()
239
+ print("Maya AI ready!")
240
 
241
  # Gradio interface functions
242
  def process_audio(audio):
243
  if audio is None:
244
+ return "Please record some audio first", None, "Click the microphone button above to start recording"
245
 
246
+ return ai_system.process_conversation(audio)
 
247
 
248
  def clear_chat():
249
  message = ai_system.clear_conversation()
250
+ return "", None, message
251
+
252
+ def greet():
253
+ return "", None, "πŸ‘‹ Hi! I'm Maya, your AI conversation partner. Click the microphone button and start talking!"
254
 
255
  # Create Gradio interface
256
+ with gr.Blocks(
257
+ title="Maya AI - Conversational Assistant",
258
+ theme=gr.themes.Soft(),
259
+ css="""
260
+ .gradio-container {
261
+ max-width: 1200px !important;
262
+ }
263
+ .audio-container {
264
+ min-height: 200px;
265
+ }
266
+ """
267
+ ) as demo:
268
+
269
+ gr.Markdown("""
270
+ # 🎀 Maya AI - Your Conversational Partner
271
+ *Advanced speech recognition with emotional understanding*
272
+
273
+ **Instructions:** Click the microphone button, speak clearly, then click stop. Maya will respond with voice and text!
274
+ """)
275
 
276
  with gr.Row():
277
  with gr.Column(scale=1):
278
+ gr.Markdown("### πŸŽ™οΈ Voice Input")
279
  audio_input = gr.Audio(
280
  sources=["microphone"],
281
  type="filepath",
282
+ label="Record your message",
283
+ elem_classes=["audio-container"]
284
  )
285
 
286
+ with gr.Row():
287
+ process_btn = gr.Button("πŸ’¬ Process Audio", variant="primary", size="lg")
288
+ clear_btn = gr.Button("πŸ—‘οΈ Clear Chat", variant="secondary")
289
 
290
  with gr.Column(scale=2):
291
+ gr.Markdown("### πŸ“ Conversation")
292
  transcription_output = gr.Textbox(
293
+ label="What you said",
294
  lines=2,
295
+ interactive=False,
296
+ placeholder="Your speech will appear here..."
297
  )
298
 
299
  audio_output = gr.Audio(
300
  label="πŸ”Š Maya's Response",
301
+ interactive=False,
302
+ autoplay=True
303
  )
304
 
305
  conversation_history = gr.Textbox(
306
  label="πŸ’­ Conversation History",
307
+ lines=12,
308
+ interactive=False,
309
+ placeholder="Conversation history will appear here...",
310
+ show_copy_button=True
311
  )
312
 
313
  # Event handlers
 
319
 
320
  clear_btn.click(
321
  fn=clear_chat,
322
+ outputs=[transcription_output, audio_output, conversation_history]
323
  )
324
 
325
+ # Auto-process when audio is uploaded/recorded
326
+ audio_input.stop_recording(
327
  fn=process_audio,
328
  inputs=[audio_input],
329
  outputs=[transcription_output, audio_output, conversation_history]
330
  )
331
+
332
+ # Initialize with greeting
333
+ demo.load(
334
+ fn=greet,
335
+ outputs=[transcription_output, audio_output, conversation_history]
336
+ )
337
 
338
  # Launch the app
339
  if __name__ == "__main__":
340
+ demo.launch(
341
+ server_name="0.0.0.0",
342
+ server_port=7860,
343
+ show_error=True,
344
+ show_tips=False,
345
+ quiet=True
346
+ )