Spaces:
Runtime error
Runtime error
File size: 12,710 Bytes
9ce0951 bd093cc | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 | import io
import os
from pathlib import Path
from typing import Dict, Tuple
import librosa
import numpy as np
import speech_recognition as sr
import soundfile as sf
from .llm_service import get_llm
from langchain.prompts import PromptTemplate
# Constants
MAX_FILE_SIZE = 25 * 1024 * 1024 # 25MB
SUPPORTED_FORMATS = {'.wav', '.mp3', '.m4a'}
MAX_DURATION = 300 # 5 minutes in seconds
class AudioAnalyzer:
def __init__(self):
self.recognizer = sr.Recognizer()
def validate_audio_file(self, file_path: Path) -> None:
"""Validate audio file format and size"""
if not file_path.suffix.lower() in SUPPORTED_FORMATS:
raise ValueError(f"Unsupported audio format. Supported formats: {', '.join(SUPPORTED_FORMATS)}")
if os.path.getsize(file_path) > MAX_FILE_SIZE:
raise ValueError(f"File size exceeds maximum limit of {MAX_FILE_SIZE/1024/1024:.1f}MB")
def _extract_audio_features(self, audio_data: bytes) -> Tuple[np.ndarray, float]:
"""Extract audio features using librosa"""
try:
audio_array, sample_rate = sf.read(io.BytesIO(audio_data))
# If stereo, convert to mono
if len(audio_array.shape) > 1:
audio_array = np.mean(audio_array, axis=1)
# Check duration
duration = len(audio_array) / sample_rate
if duration > MAX_DURATION:
raise ValueError(f"Audio duration exceeds maximum limit of {MAX_DURATION/60:.1f} minutes")
return audio_array, sample_rate
except Exception as e:
raise Exception(f"Failed to process audio file: {e}")
def _transcribe_audio(self, audio_array: np.ndarray, sample_rate: float) -> str:
"""Transcribe audio using speech recognition"""
with io.BytesIO() as wav_io:
sf.write(wav_io, audio_array, sample_rate, format='wav')
wav_io.seek(0)
with sr.AudioFile(wav_io) as source:
audio_data = self.recognizer.record(source)
try:
return self.recognizer.recognize_google(audio_data)
except sr.UnknownValueError:
return "Speech Recognition could not understand the audio"
except sr.RequestError:
return "Could not request results from Speech Recognition service"
def _calculate_fluency_score(self, audio_array: np.ndarray, sample_rate: float) -> float:
"""Calculate fluency score based on silence intervals and speech rate"""
# Get onsets for words
onset_env = librosa.onset.onset_strength(y=audio_array, sr=sample_rate)
tempo, _ = librosa.beat.beat_track(onset_envelope=onset_env, sr=sample_rate)
# Calculate speech rate normalized to a 0-1 scale
optimal_tempo = 150 # Increased from 120 to be more lenient
tempo_score = 1 - min(1.0, abs(tempo - optimal_tempo) / optimal_tempo)
# Calculate silence ratio with more lenient threshold
silence_threshold = 0.02 # Reduced from 0.03 to be more lenient
is_silence = audio_array < silence_threshold
silence_ratio = np.mean(is_silence)
# Combine metrics with adjusted weights
fluency_score = (tempo_score * 0.8 + (1 - silence_ratio) * 0.2) # Increased weight on tempo
return min(1.0, max(0.0, fluency_score))
def _analyze_tone(self, transcribed_text: str) -> Dict:
"""Analyze tone based on text characteristics"""
# Analyze based on punctuation and sentence structure
sentences = [s.strip() for s in transcribed_text.split('.') if s.strip()]
avg_sentence_length = np.mean([len(s.split()) for s in sentences]) if sentences else 0
# Simple tone estimation based on sentence characteristics
tone_score = min(1.0, avg_sentence_length / 20) # Normalize to 0-1
return {
"overall_tone": float(tone_score)
}
def _calculate_confidence_score(self, audio_array: np.ndarray, transcribed_text: str) -> float:
"""Calculate confidence score based on audio characteristics and text"""
# Audio energy variation with more lenient thresholds
rmse = librosa.feature.rms(y=audio_array)[0]
energy_std = np.std(rmse)
energy_score = min(1.0, energy_std * 20)
# Speech rate consistency with adjusted parameters
onset_env = librosa.onset.onset_strength(y=audio_array)
onset_std = np.std(onset_env)
consistency_score = 1 - min(1.0, onset_std / 10) # was 8
# Combine scores with adjusted weights
confidence_score = (energy_score * 0.7 + consistency_score * 0.3)
return min(1.0, max(0.0, confidence_score))
def _get_grammar_score(self, text: str) -> float:
"""
Calculate a basic grammar score based on sentence structure
Note: This is a simplified version. For production, use a proper grammar checker API
"""
sentences = [s.strip() for s in text.split('.') if s.strip()]
word_counts = [len(s.split()) for s in sentences]
# Analyze sentence length variation
sentence_length_std = np.std(word_counts) if word_counts else 0
length_score = 1 - min(1.0, sentence_length_std / 15) # higher tolerance for variance
# Count basic punctuation
punctuation_score = min(1.0, (sum(1 for c in text if c in '.,!?') / len(text)) * 2) # amplify good punctuation
return min(1.0, (length_score * 0.7 + punctuation_score * 0.3))
def _calculate_wpm(self, audio_duration: float, transcribed_text: str) -> float:
"""Calculate Words Per Minute"""
word_count = len(transcribed_text.split())
minutes = audio_duration / 60
return word_count / minutes if minutes > 0 else 0
def _get_assessment_prompt(self) -> PromptTemplate:
"""Create a prompt template for LLM assessment"""
template = '''You are an interview evaluation assistant. Based on the given metrics—fluency, confidence, grammar, tone, and pauses—generate a brief, constructive assessment of the candidate's spoken response. Use a tone that is supportive and human-like, suitable for hiring and recruitment purposes.
Audio Analysis Metrics:
- Fluency Score: {fluency_score}
- Grammar Score: {grammar_score}
- Confidence Score: {confidence_score}
- Clarity Score: {clarity_score}
- Pronunciation Score: {pronunciation_score}
- Words Per Minute: {wpm_rate}
- Filler Words Count: {filler_words_count}
- Speaking Duration: {speaking_duration}
- Number of Pauses: {pauses_count}
Transcribed Response:
"{transcribed_text}"
Job Role: {job_role}
Please provide a comprehensive assessment including:
1. Overall communication effectiveness
2. Areas of strength
3. Areas for improvement
4. Interview performance score (0-100)
5. Hiring recommendation (recommend/consider/reject)
Always mention:
1. Strengths: Start by highlighting what the candidate did well.
2. Areas for improvement: Be gentle, offering actionable suggestions.
3. Overall impression: Summarize whether the delivery is interview-ready or needs polishing.
Be encouraging and avoid sounding robotic or overly critical.
Format your response as a clear, concise paragraph that covers all these points.
Focus on being direct and actionable in your assessment.'''
return PromptTemplate(
input_variables=[
"fluency_score", "grammar_score", "confidence_score",
"clarity_score", "pronunciation_score",
"wpm_rate", "filler_words_count", "speaking_duration",
"pauses_count", "transcribed_text", "job_role"
],
template=template
)
def _get_llm_assessment(self, metrics: Dict, job_role: str) -> str:
"""Get LLM assessment of the audio analysis metrics"""
try:
llm = get_llm()
prompt = self._get_assessment_prompt()
# Prepare input variables
input_vars = {
**{k: metrics[k] for k in [
"fluency_score", "grammar_score", "confidence_score",
"clarity_score", "pronunciation_score",
"wpm_rate", "filler_words_count", "speaking_duration",
"pauses_count", "transcribed_text"
]},
"job_role": job_role or "General"
}
# Get LLM response
response = llm.invoke(prompt.format_prompt(**input_vars))
# Extract just the content from the response
response_text = response.content.strip()
if response_text.startswith("`"):
response_text = response_text.strip("`")
if response_text.startswith("json"):
response_text = response_text[4:]
return response_text.strip()
except Exception as e:
print(f"Error getting LLM assessment: {str(e)}")
# Return a basic assessment based on metrics
return f"""Basic analysis shows {metrics['speaking_duration']:.1f} seconds of speech with fluency score of {metrics['fluency_score']:.2f}.
Speech clarity is {metrics['clarity_score']:.2f} and technical accuracy is {metrics['technical_accuracy']:.2f}.
The candidate shows potential but could provide more technical details.
Interview score: {int(metrics['confidence_score'] * 100)}/100.
Recommendation: Consider for further evaluation."""
def analyze_audio(self, audio_data: bytes, job_role: str = None) -> Dict:
"""Main method to analyze audio and return a simplified response with transcription and assessment"""
try:
# Extract audio features
audio_array, sample_rate = self._extract_audio_features(audio_data)
duration = len(audio_array) / sample_rate
# Get transcription
transcribed_text = self._transcribe_audio(audio_array, sample_rate)
# Calculate various scores internally
fluency_score = self._calculate_fluency_score(audio_array, sample_rate)
grammar_score = self._get_grammar_score(transcribed_text)
confidence_score = self._calculate_confidence_score(audio_array, transcribed_text)
tone_analysis = self._analyze_tone(transcribed_text)
wpm = self._calculate_wpm(duration, transcribed_text)
# Calculate clarity score based on audio features with adjusted parameters
onset_env = librosa.onset.onset_strength(y=audio_array, sr=sample_rate)
clarity_score = min(1.0, np.mean(onset_env) / 8)
# Count pauses with more lenient parameters
silence_intervals = librosa.effects.split(audio_array, top_db=35)
pauses_count = len(silence_intervals) - 1
# Simplified filler words count with reduced list
filler_words = ["um", "uh"] # Reduced list to only count significant fillers
filler_count = sum(transcribed_text.lower().count(word) for word in filler_words)
# Create metrics dictionary for LLM assessment
metrics = {
"fluency_score": float(fluency_score),
"grammar_score": float(grammar_score),
"confidence_score": float(confidence_score),
"clarity_score": float(clarity_score),
"pronunciation_score": float(clarity_score * 0.7 + fluency_score * 0.3),
"transcribed_text": transcribed_text,
"wpm_rate": float(wpm),
"filler_words_count": filler_count,
"speaking_duration": float(duration),
"pauses_count": pauses_count
}
# Get LLM assessment
assessment = self._get_llm_assessment(metrics, job_role)
# Return simplified response
return {
"transcript": transcribed_text,
"assessment": assessment
}
except Exception as e:
print(f"Error in analyze_audio: {str(e)}")
raise AudioAnalysisError(f"Failed to analyze audio: {str(e)}") |