FastAPIExperience2 / services /voice_service.py
utkarsh134's picture
Update services/voice_service.py
9ce0951 verified
Raw History Blame Contribute Delete
12.7 kB
import io
import os
from pathlib import Path
from typing import Dict, Tuple
import librosa
import numpy as np
import speech_recognition as sr
import soundfile as sf
from .llm_service import get_llm
from langchain.prompts import PromptTemplate
# Constants
MAX_FILE_SIZE = 25 * 1024 * 1024 # 25MB
SUPPORTED_FORMATS = {'.wav', '.mp3', '.m4a'}
MAX_DURATION = 300 # 5 minutes in seconds
class AudioAnalyzer:
def __init__(self):
self.recognizer = sr.Recognizer()
def validate_audio_file(self, file_path: Path) -> None:
"""Validate audio file format and size"""
if not file_path.suffix.lower() in SUPPORTED_FORMATS:
raise ValueError(f"Unsupported audio format. Supported formats: {', '.join(SUPPORTED_FORMATS)}")
if os.path.getsize(file_path) > MAX_FILE_SIZE:
raise ValueError(f"File size exceeds maximum limit of {MAX_FILE_SIZE/1024/1024:.1f}MB")
def _extract_audio_features(self, audio_data: bytes) -> Tuple[np.ndarray, float]:
"""Extract audio features using librosa"""
try:
audio_array, sample_rate = sf.read(io.BytesIO(audio_data))
# If stereo, convert to mono
if len(audio_array.shape) > 1:
audio_array = np.mean(audio_array, axis=1)
# Check duration
duration = len(audio_array) / sample_rate
if duration > MAX_DURATION:
raise ValueError(f"Audio duration exceeds maximum limit of {MAX_DURATION/60:.1f} minutes")
return audio_array, sample_rate
except Exception as e:
raise Exception(f"Failed to process audio file: {e}")
def _transcribe_audio(self, audio_array: np.ndarray, sample_rate: float) -> str:
"""Transcribe audio using speech recognition"""
with io.BytesIO() as wav_io:
sf.write(wav_io, audio_array, sample_rate, format='wav')
wav_io.seek(0)
with sr.AudioFile(wav_io) as source:
audio_data = self.recognizer.record(source)
try:
return self.recognizer.recognize_google(audio_data)
except sr.UnknownValueError:
return "Speech Recognition could not understand the audio"
except sr.RequestError:
return "Could not request results from Speech Recognition service"
def _calculate_fluency_score(self, audio_array: np.ndarray, sample_rate: float) -> float:
"""Calculate fluency score based on silence intervals and speech rate"""
# Get onsets for words
onset_env = librosa.onset.onset_strength(y=audio_array, sr=sample_rate)
tempo, _ = librosa.beat.beat_track(onset_envelope=onset_env, sr=sample_rate)
# Calculate speech rate normalized to a 0-1 scale
optimal_tempo = 150 # Increased from 120 to be more lenient
tempo_score = 1 - min(1.0, abs(tempo - optimal_tempo) / optimal_tempo)
# Calculate silence ratio with more lenient threshold
silence_threshold = 0.02 # Reduced from 0.03 to be more lenient
is_silence = audio_array < silence_threshold
silence_ratio = np.mean(is_silence)
# Combine metrics with adjusted weights
fluency_score = (tempo_score * 0.8 + (1 - silence_ratio) * 0.2) # Increased weight on tempo
return min(1.0, max(0.0, fluency_score))
def _analyze_tone(self, transcribed_text: str) -> Dict:
"""Analyze tone based on text characteristics"""
# Analyze based on punctuation and sentence structure
sentences = [s.strip() for s in transcribed_text.split('.') if s.strip()]
avg_sentence_length = np.mean([len(s.split()) for s in sentences]) if sentences else 0
# Simple tone estimation based on sentence characteristics
tone_score = min(1.0, avg_sentence_length / 20) # Normalize to 0-1
return {
"overall_tone": float(tone_score)
}
def _calculate_confidence_score(self, audio_array: np.ndarray, transcribed_text: str) -> float:
"""Calculate confidence score based on audio characteristics and text"""
# Audio energy variation with more lenient thresholds
rmse = librosa.feature.rms(y=audio_array)[0]
energy_std = np.std(rmse)
energy_score = min(1.0, energy_std * 20)
# Speech rate consistency with adjusted parameters
onset_env = librosa.onset.onset_strength(y=audio_array)
onset_std = np.std(onset_env)
consistency_score = 1 - min(1.0, onset_std / 10) # was 8
# Combine scores with adjusted weights
confidence_score = (energy_score * 0.7 + consistency_score * 0.3)
return min(1.0, max(0.0, confidence_score))
def _get_grammar_score(self, text: str) -> float:
"""
Calculate a basic grammar score based on sentence structure
Note: This is a simplified version. For production, use a proper grammar checker API
"""
sentences = [s.strip() for s in text.split('.') if s.strip()]
word_counts = [len(s.split()) for s in sentences]
# Analyze sentence length variation
sentence_length_std = np.std(word_counts) if word_counts else 0
length_score = 1 - min(1.0, sentence_length_std / 15) # higher tolerance for variance
# Count basic punctuation
punctuation_score = min(1.0, (sum(1 for c in text if c in '.,!?') / len(text)) * 2) # amplify good punctuation
return min(1.0, (length_score * 0.7 + punctuation_score * 0.3))
def _calculate_wpm(self, audio_duration: float, transcribed_text: str) -> float:
"""Calculate Words Per Minute"""
word_count = len(transcribed_text.split())
minutes = audio_duration / 60
return word_count / minutes if minutes > 0 else 0
def _get_assessment_prompt(self) -> PromptTemplate:
"""Create a prompt template for LLM assessment"""
template = '''You are an interview evaluation assistant. Based on the given metrics—fluency, confidence, grammar, tone, and pauses—generate a brief, constructive assessment of the candidate's spoken response. Use a tone that is supportive and human-like, suitable for hiring and recruitment purposes.
Audio Analysis Metrics:
- Fluency Score: {fluency_score}
- Grammar Score: {grammar_score}
- Confidence Score: {confidence_score}
- Clarity Score: {clarity_score}
- Pronunciation Score: {pronunciation_score}
- Words Per Minute: {wpm_rate}
- Filler Words Count: {filler_words_count}
- Speaking Duration: {speaking_duration}
- Number of Pauses: {pauses_count}
Transcribed Response:
"{transcribed_text}"
Job Role: {job_role}
Please provide a comprehensive assessment including:
1. Overall communication effectiveness
2. Areas of strength
3. Areas for improvement
4. Interview performance score (0-100)
5. Hiring recommendation (recommend/consider/reject)
Always mention:
1. Strengths: Start by highlighting what the candidate did well.
2. Areas for improvement: Be gentle, offering actionable suggestions.
3. Overall impression: Summarize whether the delivery is interview-ready or needs polishing.
Be encouraging and avoid sounding robotic or overly critical.
Format your response as a clear, concise paragraph that covers all these points.
Focus on being direct and actionable in your assessment.'''
return PromptTemplate(
input_variables=[
"fluency_score", "grammar_score", "confidence_score",
"clarity_score", "pronunciation_score",
"wpm_rate", "filler_words_count", "speaking_duration",
"pauses_count", "transcribed_text", "job_role"
],
template=template
)
def _get_llm_assessment(self, metrics: Dict, job_role: str) -> str:
"""Get LLM assessment of the audio analysis metrics"""
try:
llm = get_llm()
prompt = self._get_assessment_prompt()
# Prepare input variables
input_vars = {
**{k: metrics[k] for k in [
"fluency_score", "grammar_score", "confidence_score",
"clarity_score", "pronunciation_score",
"wpm_rate", "filler_words_count", "speaking_duration",
"pauses_count", "transcribed_text"
]},
"job_role": job_role or "General"
}
# Get LLM response
response = llm.invoke(prompt.format_prompt(**input_vars))
# Extract just the content from the response
response_text = response.content.strip()
if response_text.startswith("`"):
response_text = response_text.strip("`")
if response_text.startswith("json"):
response_text = response_text[4:]
return response_text.strip()
except Exception as e:
print(f"Error getting LLM assessment: {str(e)}")
# Return a basic assessment based on metrics
return f"""Basic analysis shows {metrics['speaking_duration']:.1f} seconds of speech with fluency score of {metrics['fluency_score']:.2f}.
Speech clarity is {metrics['clarity_score']:.2f} and technical accuracy is {metrics['technical_accuracy']:.2f}.
The candidate shows potential but could provide more technical details.
Interview score: {int(metrics['confidence_score'] * 100)}/100.
Recommendation: Consider for further evaluation."""
def analyze_audio(self, audio_data: bytes, job_role: str = None) -> Dict:
"""Main method to analyze audio and return a simplified response with transcription and assessment"""
try:
# Extract audio features
audio_array, sample_rate = self._extract_audio_features(audio_data)
duration = len(audio_array) / sample_rate
# Get transcription
transcribed_text = self._transcribe_audio(audio_array, sample_rate)
# Calculate various scores internally
fluency_score = self._calculate_fluency_score(audio_array, sample_rate)
grammar_score = self._get_grammar_score(transcribed_text)
confidence_score = self._calculate_confidence_score(audio_array, transcribed_text)
tone_analysis = self._analyze_tone(transcribed_text)
wpm = self._calculate_wpm(duration, transcribed_text)
# Calculate clarity score based on audio features with adjusted parameters
onset_env = librosa.onset.onset_strength(y=audio_array, sr=sample_rate)
clarity_score = min(1.0, np.mean(onset_env) / 8)
# Count pauses with more lenient parameters
silence_intervals = librosa.effects.split(audio_array, top_db=35)
pauses_count = len(silence_intervals) - 1
# Simplified filler words count with reduced list
filler_words = ["um", "uh"] # Reduced list to only count significant fillers
filler_count = sum(transcribed_text.lower().count(word) for word in filler_words)
# Create metrics dictionary for LLM assessment
metrics = {
"fluency_score": float(fluency_score),
"grammar_score": float(grammar_score),
"confidence_score": float(confidence_score),
"clarity_score": float(clarity_score),
"pronunciation_score": float(clarity_score * 0.7 + fluency_score * 0.3),
"transcribed_text": transcribed_text,
"wpm_rate": float(wpm),
"filler_words_count": filler_count,
"speaking_duration": float(duration),
"pauses_count": pauses_count
}
# Get LLM assessment
assessment = self._get_llm_assessment(metrics, job_role)
# Return simplified response
return {
"transcript": transcribed_text,
"assessment": assessment
}
except Exception as e:
print(f"Error in analyze_audio: {str(e)}")
raise AudioAnalysisError(f"Failed to analyze audio: {str(e)}")