File size: 12,710 Bytes
9ce0951
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
bd093cc
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
import io
import os
from pathlib import Path
from typing import Dict, Tuple
import librosa
import numpy as np
import speech_recognition as sr
import soundfile as sf
from .llm_service import get_llm
from langchain.prompts import PromptTemplate

# Constants
MAX_FILE_SIZE = 25 * 1024 * 1024  # 25MB
SUPPORTED_FORMATS = {'.wav', '.mp3', '.m4a'}
MAX_DURATION = 300  # 5 minutes in seconds

class AudioAnalyzer:
    def __init__(self):
        self.recognizer = sr.Recognizer()

    def validate_audio_file(self, file_path: Path) -> None:
        """Validate audio file format and size"""
        if not file_path.suffix.lower() in SUPPORTED_FORMATS:
            raise ValueError(f"Unsupported audio format. Supported formats: {', '.join(SUPPORTED_FORMATS)}")
        
        if os.path.getsize(file_path) > MAX_FILE_SIZE:
            raise ValueError(f"File size exceeds maximum limit of {MAX_FILE_SIZE/1024/1024:.1f}MB")

    def _extract_audio_features(self, audio_data: bytes) -> Tuple[np.ndarray, float]:
        """Extract audio features using librosa"""
        try:
            audio_array, sample_rate = sf.read(io.BytesIO(audio_data))
            
            # If stereo, convert to mono
            if len(audio_array.shape) > 1:
                audio_array = np.mean(audio_array, axis=1)

            # Check duration
            duration = len(audio_array) / sample_rate
            if duration > MAX_DURATION:
                raise ValueError(f"Audio duration exceeds maximum limit of {MAX_DURATION/60:.1f} minutes")
                
            return audio_array, sample_rate
        except Exception as e:
            raise Exception(f"Failed to process audio file: {e}")

    def _transcribe_audio(self, audio_array: np.ndarray, sample_rate: float) -> str:
        """Transcribe audio using speech recognition"""
        with io.BytesIO() as wav_io:
            sf.write(wav_io, audio_array, sample_rate, format='wav')
            wav_io.seek(0)
            with sr.AudioFile(wav_io) as source:
                audio_data = self.recognizer.record(source)
                try:
                    return self.recognizer.recognize_google(audio_data)
                except sr.UnknownValueError:
                    return "Speech Recognition could not understand the audio"
                except sr.RequestError:
                    return "Could not request results from Speech Recognition service"

    def _calculate_fluency_score(self, audio_array: np.ndarray, sample_rate: float) -> float:
        """Calculate fluency score based on silence intervals and speech rate"""
        # Get onsets for words
        onset_env = librosa.onset.onset_strength(y=audio_array, sr=sample_rate)
        tempo, _ = librosa.beat.beat_track(onset_envelope=onset_env, sr=sample_rate)
        
        # Calculate speech rate normalized to a 0-1 scale
        optimal_tempo = 150  # Increased from 120 to be more lenient
        tempo_score = 1 - min(1.0, abs(tempo - optimal_tempo) / optimal_tempo)
        
        # Calculate silence ratio with more lenient threshold
        silence_threshold = 0.02  # Reduced from 0.03 to be more lenient
        is_silence = audio_array < silence_threshold
        silence_ratio = np.mean(is_silence)
        
        # Combine metrics with adjusted weights
        fluency_score = (tempo_score * 0.8 + (1 - silence_ratio) * 0.2)  # Increased weight on tempo
        return min(1.0, max(0.0, fluency_score))

    def _analyze_tone(self, transcribed_text: str) -> Dict:
        """Analyze tone based on text characteristics"""
        # Analyze based on punctuation and sentence structure
        sentences = [s.strip() for s in transcribed_text.split('.') if s.strip()]
        avg_sentence_length = np.mean([len(s.split()) for s in sentences]) if sentences else 0
        
        # Simple tone estimation based on sentence characteristics
        tone_score = min(1.0, avg_sentence_length / 20)  # Normalize to 0-1        
        return {
            "overall_tone": float(tone_score)
        }

    def _calculate_confidence_score(self, audio_array: np.ndarray, transcribed_text: str) -> float:
        """Calculate confidence score based on audio characteristics and text"""
        # Audio energy variation with more lenient thresholds
        rmse = librosa.feature.rms(y=audio_array)[0]
        energy_std = np.std(rmse)
        energy_score = min(1.0, energy_std * 20) 
        
        # Speech rate consistency with adjusted parameters
        onset_env = librosa.onset.onset_strength(y=audio_array)
        onset_std = np.std(onset_env)
        consistency_score = 1 - min(1.0, onset_std / 10)  # was 8

        
        # Combine scores with adjusted weights
        confidence_score = (energy_score * 0.7 + consistency_score * 0.3)
        return min(1.0, max(0.0, confidence_score))

    def _get_grammar_score(self, text: str) -> float:
        """
        Calculate a basic grammar score based on sentence structure
        Note: This is a simplified version. For production, use a proper grammar checker API
        """
        sentences = [s.strip() for s in text.split('.') if s.strip()]
        word_counts = [len(s.split()) for s in sentences]
        
        # Analyze sentence length variation
        sentence_length_std = np.std(word_counts) if word_counts else 0
        length_score = 1 - min(1.0, sentence_length_std / 15)  # higher tolerance for variance
        
        # Count basic punctuation
        punctuation_score = min(1.0, (sum(1 for c in text if c in '.,!?') / len(text)) * 2)  # amplify good punctuation
        
        return min(1.0, (length_score * 0.7 + punctuation_score * 0.3))

    def _calculate_wpm(self, audio_duration: float, transcribed_text: str) -> float:
        """Calculate Words Per Minute"""
        word_count = len(transcribed_text.split())
        minutes = audio_duration / 60
        return word_count / minutes if minutes > 0 else 0


    def _get_assessment_prompt(self) -> PromptTemplate:
        """Create a prompt template for LLM assessment"""
        template = '''You are an interview evaluation assistant. Based on the given metrics—fluency, confidence, grammar, tone, and pauses—generate a brief, constructive assessment of the candidate's spoken response. Use a tone that is supportive and human-like, suitable for hiring and recruitment purposes.

        Audio Analysis Metrics:
        - Fluency Score: {fluency_score}
        - Grammar Score: {grammar_score}
        - Confidence Score: {confidence_score}
        - Clarity Score: {clarity_score}
        - Pronunciation Score: {pronunciation_score}
        - Words Per Minute: {wpm_rate}
        - Filler Words Count: {filler_words_count}
        - Speaking Duration: {speaking_duration}
        - Number of Pauses: {pauses_count}

        Transcribed Response:
        "{transcribed_text}"

        Job Role: {job_role}

        Please provide a comprehensive assessment including:
        1. Overall communication effectiveness
        2. Areas of strength
        3. Areas for improvement
        4. Interview performance score (0-100)
        5. Hiring recommendation (recommend/consider/reject)
        
        Always mention:
        1. Strengths: Start by highlighting what the candidate did well.
        2. Areas for improvement: Be gentle, offering actionable suggestions.
        3. Overall impression: Summarize whether the delivery is interview-ready or needs polishing.

        Be encouraging and avoid sounding robotic or overly critical.
        Format your response as a clear, concise paragraph that covers all these points.
        Focus on being direct and actionable in your assessment.'''
        return PromptTemplate(
            input_variables=[
                "fluency_score", "grammar_score", "confidence_score",
                "clarity_score", "pronunciation_score",
                "wpm_rate", "filler_words_count", "speaking_duration",
                "pauses_count", "transcribed_text", "job_role"
            ],
            template=template
        )

    def _get_llm_assessment(self, metrics: Dict, job_role: str) -> str:
        """Get LLM assessment of the audio analysis metrics"""
        try:
            llm = get_llm()
            prompt = self._get_assessment_prompt()
            
            # Prepare input variables
            input_vars = {
                **{k: metrics[k] for k in [
                    "fluency_score", "grammar_score", "confidence_score",
                    "clarity_score", "pronunciation_score",
                    "wpm_rate", "filler_words_count", "speaking_duration",
                    "pauses_count", "transcribed_text"
                ]},
                "job_role": job_role or "General"
            }
            
            # Get LLM response
            response = llm.invoke(prompt.format_prompt(**input_vars))
            
            # Extract just the content from the response
            response_text = response.content.strip()
            if response_text.startswith("`"):
                response_text = response_text.strip("`")
            if response_text.startswith("json"):
                response_text = response_text[4:]
            
            return response_text.strip()
                
        except Exception as e:
            print(f"Error getting LLM assessment: {str(e)}")
            # Return a basic assessment based on metrics
            return f"""Basic analysis shows {metrics['speaking_duration']:.1f} seconds of speech with fluency score of {metrics['fluency_score']:.2f}. 
            Speech clarity is {metrics['clarity_score']:.2f} and technical accuracy is {metrics['technical_accuracy']:.2f}. 
            The candidate shows potential but could provide more technical details. 
            Interview score: {int(metrics['confidence_score'] * 100)}/100. 
            Recommendation: Consider for further evaluation."""

    def analyze_audio(self, audio_data: bytes, job_role: str = None) -> Dict:
        """Main method to analyze audio and return a simplified response with transcription and assessment"""
        try:
            # Extract audio features
            audio_array, sample_rate = self._extract_audio_features(audio_data)
            duration = len(audio_array) / sample_rate
            
            # Get transcription
            transcribed_text = self._transcribe_audio(audio_array, sample_rate)
            
            # Calculate various scores internally
            fluency_score = self._calculate_fluency_score(audio_array, sample_rate)
            grammar_score = self._get_grammar_score(transcribed_text)
            confidence_score = self._calculate_confidence_score(audio_array, transcribed_text)
            tone_analysis = self._analyze_tone(transcribed_text)
            wpm = self._calculate_wpm(duration, transcribed_text)
            
            # Calculate clarity score based on audio features with adjusted parameters
            onset_env = librosa.onset.onset_strength(y=audio_array, sr=sample_rate)
            clarity_score = min(1.0, np.mean(onset_env) / 8)
            
            # Count pauses with more lenient parameters
            silence_intervals = librosa.effects.split(audio_array, top_db=35)
            pauses_count = len(silence_intervals) - 1
            
            # Simplified filler words count with reduced list
            filler_words = ["um", "uh"]  # Reduced list to only count significant fillers
            filler_count = sum(transcribed_text.lower().count(word) for word in filler_words)
            
            # Create metrics dictionary for LLM assessment
            metrics = {
                "fluency_score": float(fluency_score),
                "grammar_score": float(grammar_score),
                "confidence_score": float(confidence_score),
                "clarity_score": float(clarity_score),
                "pronunciation_score": float(clarity_score * 0.7 + fluency_score * 0.3),
                "transcribed_text": transcribed_text,
                "wpm_rate": float(wpm),
                "filler_words_count": filler_count,            
                "speaking_duration": float(duration),
                "pauses_count": pauses_count
            }
            
            # Get LLM assessment
            assessment = self._get_llm_assessment(metrics, job_role)
            
            # Return simplified response
            return {
                "transcript": transcribed_text,
                "assessment": assessment
            }
        except Exception as e:
            print(f"Error in analyze_audio: {str(e)}")
            raise AudioAnalysisError(f"Failed to analyze audio: {str(e)}")