import io import os from pathlib import Path from typing import Dict, Tuple import librosa import numpy as np import speech_recognition as sr import soundfile as sf from .llm_service import get_llm from langchain.prompts import PromptTemplate # Constants MAX_FILE_SIZE = 25 * 1024 * 1024 # 25MB SUPPORTED_FORMATS = {'.wav', '.mp3', '.m4a'} MAX_DURATION = 300 # 5 minutes in seconds class AudioAnalyzer: def __init__(self): self.recognizer = sr.Recognizer() def validate_audio_file(self, file_path: Path) -> None: """Validate audio file format and size""" if not file_path.suffix.lower() in SUPPORTED_FORMATS: raise ValueError(f"Unsupported audio format. Supported formats: {', '.join(SUPPORTED_FORMATS)}") if os.path.getsize(file_path) > MAX_FILE_SIZE: raise ValueError(f"File size exceeds maximum limit of {MAX_FILE_SIZE/1024/1024:.1f}MB") def _extract_audio_features(self, audio_data: bytes) -> Tuple[np.ndarray, float]: """Extract audio features using librosa""" try: audio_array, sample_rate = sf.read(io.BytesIO(audio_data)) # If stereo, convert to mono if len(audio_array.shape) > 1: audio_array = np.mean(audio_array, axis=1) # Check duration duration = len(audio_array) / sample_rate if duration > MAX_DURATION: raise ValueError(f"Audio duration exceeds maximum limit of {MAX_DURATION/60:.1f} minutes") return audio_array, sample_rate except Exception as e: raise Exception(f"Failed to process audio file: {e}") def _transcribe_audio(self, audio_array: np.ndarray, sample_rate: float) -> str: """Transcribe audio using speech recognition""" with io.BytesIO() as wav_io: sf.write(wav_io, audio_array, sample_rate, format='wav') wav_io.seek(0) with sr.AudioFile(wav_io) as source: audio_data = self.recognizer.record(source) try: return self.recognizer.recognize_google(audio_data) except sr.UnknownValueError: return "Speech Recognition could not understand the audio" except sr.RequestError: return "Could not request results from Speech Recognition service" def _calculate_fluency_score(self, audio_array: np.ndarray, sample_rate: float) -> float: """Calculate fluency score based on silence intervals and speech rate""" # Get onsets for words onset_env = librosa.onset.onset_strength(y=audio_array, sr=sample_rate) tempo, _ = librosa.beat.beat_track(onset_envelope=onset_env, sr=sample_rate) # Calculate speech rate normalized to a 0-1 scale optimal_tempo = 150 # Increased from 120 to be more lenient tempo_score = 1 - min(1.0, abs(tempo - optimal_tempo) / optimal_tempo) # Calculate silence ratio with more lenient threshold silence_threshold = 0.02 # Reduced from 0.03 to be more lenient is_silence = audio_array < silence_threshold silence_ratio = np.mean(is_silence) # Combine metrics with adjusted weights fluency_score = (tempo_score * 0.8 + (1 - silence_ratio) * 0.2) # Increased weight on tempo return min(1.0, max(0.0, fluency_score)) def _analyze_tone(self, transcribed_text: str) -> Dict: """Analyze tone based on text characteristics""" # Analyze based on punctuation and sentence structure sentences = [s.strip() for s in transcribed_text.split('.') if s.strip()] avg_sentence_length = np.mean([len(s.split()) for s in sentences]) if sentences else 0 # Simple tone estimation based on sentence characteristics tone_score = min(1.0, avg_sentence_length / 20) # Normalize to 0-1 return { "overall_tone": float(tone_score) } def _calculate_confidence_score(self, audio_array: np.ndarray, transcribed_text: str) -> float: """Calculate confidence score based on audio characteristics and text""" # Audio energy variation with more lenient thresholds rmse = librosa.feature.rms(y=audio_array)[0] energy_std = np.std(rmse) energy_score = min(1.0, energy_std * 20) # Speech rate consistency with adjusted parameters onset_env = librosa.onset.onset_strength(y=audio_array) onset_std = np.std(onset_env) consistency_score = 1 - min(1.0, onset_std / 10) # was 8 # Combine scores with adjusted weights confidence_score = (energy_score * 0.7 + consistency_score * 0.3) return min(1.0, max(0.0, confidence_score)) def _get_grammar_score(self, text: str) -> float: """ Calculate a basic grammar score based on sentence structure Note: This is a simplified version. For production, use a proper grammar checker API """ sentences = [s.strip() for s in text.split('.') if s.strip()] word_counts = [len(s.split()) for s in sentences] # Analyze sentence length variation sentence_length_std = np.std(word_counts) if word_counts else 0 length_score = 1 - min(1.0, sentence_length_std / 15) # higher tolerance for variance # Count basic punctuation punctuation_score = min(1.0, (sum(1 for c in text if c in '.,!?') / len(text)) * 2) # amplify good punctuation return min(1.0, (length_score * 0.7 + punctuation_score * 0.3)) def _calculate_wpm(self, audio_duration: float, transcribed_text: str) -> float: """Calculate Words Per Minute""" word_count = len(transcribed_text.split()) minutes = audio_duration / 60 return word_count / minutes if minutes > 0 else 0 def _get_assessment_prompt(self) -> PromptTemplate: """Create a prompt template for LLM assessment""" template = '''You are an interview evaluation assistant. Based on the given metrics—fluency, confidence, grammar, tone, and pauses—generate a brief, constructive assessment of the candidate's spoken response. Use a tone that is supportive and human-like, suitable for hiring and recruitment purposes. Audio Analysis Metrics: - Fluency Score: {fluency_score} - Grammar Score: {grammar_score} - Confidence Score: {confidence_score} - Clarity Score: {clarity_score} - Pronunciation Score: {pronunciation_score} - Words Per Minute: {wpm_rate} - Filler Words Count: {filler_words_count} - Speaking Duration: {speaking_duration} - Number of Pauses: {pauses_count} Transcribed Response: "{transcribed_text}" Job Role: {job_role} Please provide a comprehensive assessment including: 1. Overall communication effectiveness 2. Areas of strength 3. Areas for improvement 4. Interview performance score (0-100) 5. Hiring recommendation (recommend/consider/reject) Always mention: 1. Strengths: Start by highlighting what the candidate did well. 2. Areas for improvement: Be gentle, offering actionable suggestions. 3. Overall impression: Summarize whether the delivery is interview-ready or needs polishing. Be encouraging and avoid sounding robotic or overly critical. Format your response as a clear, concise paragraph that covers all these points. Focus on being direct and actionable in your assessment.''' return PromptTemplate( input_variables=[ "fluency_score", "grammar_score", "confidence_score", "clarity_score", "pronunciation_score", "wpm_rate", "filler_words_count", "speaking_duration", "pauses_count", "transcribed_text", "job_role" ], template=template ) def _get_llm_assessment(self, metrics: Dict, job_role: str) -> str: """Get LLM assessment of the audio analysis metrics""" try: llm = get_llm() prompt = self._get_assessment_prompt() # Prepare input variables input_vars = { **{k: metrics[k] for k in [ "fluency_score", "grammar_score", "confidence_score", "clarity_score", "pronunciation_score", "wpm_rate", "filler_words_count", "speaking_duration", "pauses_count", "transcribed_text" ]}, "job_role": job_role or "General" } # Get LLM response response = llm.invoke(prompt.format_prompt(**input_vars)) # Extract just the content from the response response_text = response.content.strip() if response_text.startswith("`"): response_text = response_text.strip("`") if response_text.startswith("json"): response_text = response_text[4:] return response_text.strip() except Exception as e: print(f"Error getting LLM assessment: {str(e)}") # Return a basic assessment based on metrics return f"""Basic analysis shows {metrics['speaking_duration']:.1f} seconds of speech with fluency score of {metrics['fluency_score']:.2f}. Speech clarity is {metrics['clarity_score']:.2f} and technical accuracy is {metrics['technical_accuracy']:.2f}. The candidate shows potential but could provide more technical details. Interview score: {int(metrics['confidence_score'] * 100)}/100. Recommendation: Consider for further evaluation.""" def analyze_audio(self, audio_data: bytes, job_role: str = None) -> Dict: """Main method to analyze audio and return a simplified response with transcription and assessment""" try: # Extract audio features audio_array, sample_rate = self._extract_audio_features(audio_data) duration = len(audio_array) / sample_rate # Get transcription transcribed_text = self._transcribe_audio(audio_array, sample_rate) # Calculate various scores internally fluency_score = self._calculate_fluency_score(audio_array, sample_rate) grammar_score = self._get_grammar_score(transcribed_text) confidence_score = self._calculate_confidence_score(audio_array, transcribed_text) tone_analysis = self._analyze_tone(transcribed_text) wpm = self._calculate_wpm(duration, transcribed_text) # Calculate clarity score based on audio features with adjusted parameters onset_env = librosa.onset.onset_strength(y=audio_array, sr=sample_rate) clarity_score = min(1.0, np.mean(onset_env) / 8) # Count pauses with more lenient parameters silence_intervals = librosa.effects.split(audio_array, top_db=35) pauses_count = len(silence_intervals) - 1 # Simplified filler words count with reduced list filler_words = ["um", "uh"] # Reduced list to only count significant fillers filler_count = sum(transcribed_text.lower().count(word) for word in filler_words) # Create metrics dictionary for LLM assessment metrics = { "fluency_score": float(fluency_score), "grammar_score": float(grammar_score), "confidence_score": float(confidence_score), "clarity_score": float(clarity_score), "pronunciation_score": float(clarity_score * 0.7 + fluency_score * 0.3), "transcribed_text": transcribed_text, "wpm_rate": float(wpm), "filler_words_count": filler_count, "speaking_duration": float(duration), "pauses_count": pauses_count } # Get LLM assessment assessment = self._get_llm_assessment(metrics, job_role) # Return simplified response return { "transcript": transcribed_text, "assessment": assessment } except Exception as e: print(f"Error in analyze_audio: {str(e)}") raise AudioAnalysisError(f"Failed to analyze audio: {str(e)}")