Spaces:
Runtime error
Runtime error
Download services/voice_service.py from utkarsh134/FastAPIExperience2: direct link, hf CLI and curl.
- Browser
- Download file 12.7 kB
-
https://huggingface.co/spaces/utkarsh134/FastAPIExperience2/resolve/main/services/voice_service.py
- Command line
-
hf download hf://spaces/utkarsh134/FastAPIExperience2/services/voice_service.py
-
curl -L -o voice_service.py https://huggingface.co/spaces/utkarsh134/FastAPIExperience2/resolve/main/services/voice_service.py
12.7 kB
| import io | |
| import os | |
| from pathlib import Path | |
| from typing import Dict, Tuple | |
| import librosa | |
| import numpy as np | |
| import speech_recognition as sr | |
| import soundfile as sf | |
| from .llm_service import get_llm | |
| from langchain.prompts import PromptTemplate | |
| # Constants | |
| MAX_FILE_SIZE = 25 * 1024 * 1024 # 25MB | |
| SUPPORTED_FORMATS = {'.wav', '.mp3', '.m4a'} | |
| MAX_DURATION = 300 # 5 minutes in seconds | |
| class AudioAnalyzer: | |
| def __init__(self): | |
| self.recognizer = sr.Recognizer() | |
| def validate_audio_file(self, file_path: Path) -> None: | |
| """Validate audio file format and size""" | |
| if not file_path.suffix.lower() in SUPPORTED_FORMATS: | |
| raise ValueError(f"Unsupported audio format. Supported formats: {', '.join(SUPPORTED_FORMATS)}") | |
| if os.path.getsize(file_path) > MAX_FILE_SIZE: | |
| raise ValueError(f"File size exceeds maximum limit of {MAX_FILE_SIZE/1024/1024:.1f}MB") | |
| def _extract_audio_features(self, audio_data: bytes) -> Tuple[np.ndarray, float]: | |
| """Extract audio features using librosa""" | |
| try: | |
| audio_array, sample_rate = sf.read(io.BytesIO(audio_data)) | |
| # If stereo, convert to mono | |
| if len(audio_array.shape) > 1: | |
| audio_array = np.mean(audio_array, axis=1) | |
| # Check duration | |
| duration = len(audio_array) / sample_rate | |
| if duration > MAX_DURATION: | |
| raise ValueError(f"Audio duration exceeds maximum limit of {MAX_DURATION/60:.1f} minutes") | |
| return audio_array, sample_rate | |
| except Exception as e: | |
| raise Exception(f"Failed to process audio file: {e}") | |
| def _transcribe_audio(self, audio_array: np.ndarray, sample_rate: float) -> str: | |
| """Transcribe audio using speech recognition""" | |
| with io.BytesIO() as wav_io: | |
| sf.write(wav_io, audio_array, sample_rate, format='wav') | |
| wav_io.seek(0) | |
| with sr.AudioFile(wav_io) as source: | |
| audio_data = self.recognizer.record(source) | |
| try: | |
| return self.recognizer.recognize_google(audio_data) | |
| except sr.UnknownValueError: | |
| return "Speech Recognition could not understand the audio" | |
| except sr.RequestError: | |
| return "Could not request results from Speech Recognition service" | |
| def _calculate_fluency_score(self, audio_array: np.ndarray, sample_rate: float) -> float: | |
| """Calculate fluency score based on silence intervals and speech rate""" | |
| # Get onsets for words | |
| onset_env = librosa.onset.onset_strength(y=audio_array, sr=sample_rate) | |
| tempo, _ = librosa.beat.beat_track(onset_envelope=onset_env, sr=sample_rate) | |
| # Calculate speech rate normalized to a 0-1 scale | |
| optimal_tempo = 150 # Increased from 120 to be more lenient | |
| tempo_score = 1 - min(1.0, abs(tempo - optimal_tempo) / optimal_tempo) | |
| # Calculate silence ratio with more lenient threshold | |
| silence_threshold = 0.02 # Reduced from 0.03 to be more lenient | |
| is_silence = audio_array < silence_threshold | |
| silence_ratio = np.mean(is_silence) | |
| # Combine metrics with adjusted weights | |
| fluency_score = (tempo_score * 0.8 + (1 - silence_ratio) * 0.2) # Increased weight on tempo | |
| return min(1.0, max(0.0, fluency_score)) | |
| def _analyze_tone(self, transcribed_text: str) -> Dict: | |
| """Analyze tone based on text characteristics""" | |
| # Analyze based on punctuation and sentence structure | |
| sentences = [s.strip() for s in transcribed_text.split('.') if s.strip()] | |
| avg_sentence_length = np.mean([len(s.split()) for s in sentences]) if sentences else 0 | |
| # Simple tone estimation based on sentence characteristics | |
| tone_score = min(1.0, avg_sentence_length / 20) # Normalize to 0-1 | |
| return { | |
| "overall_tone": float(tone_score) | |
| } | |
| def _calculate_confidence_score(self, audio_array: np.ndarray, transcribed_text: str) -> float: | |
| """Calculate confidence score based on audio characteristics and text""" | |
| # Audio energy variation with more lenient thresholds | |
| rmse = librosa.feature.rms(y=audio_array)[0] | |
| energy_std = np.std(rmse) | |
| energy_score = min(1.0, energy_std * 20) | |
| # Speech rate consistency with adjusted parameters | |
| onset_env = librosa.onset.onset_strength(y=audio_array) | |
| onset_std = np.std(onset_env) | |
| consistency_score = 1 - min(1.0, onset_std / 10) # was 8 | |
| # Combine scores with adjusted weights | |
| confidence_score = (energy_score * 0.7 + consistency_score * 0.3) | |
| return min(1.0, max(0.0, confidence_score)) | |
| def _get_grammar_score(self, text: str) -> float: | |
| """ | |
| Calculate a basic grammar score based on sentence structure | |
| Note: This is a simplified version. For production, use a proper grammar checker API | |
| """ | |
| sentences = [s.strip() for s in text.split('.') if s.strip()] | |
| word_counts = [len(s.split()) for s in sentences] | |
| # Analyze sentence length variation | |
| sentence_length_std = np.std(word_counts) if word_counts else 0 | |
| length_score = 1 - min(1.0, sentence_length_std / 15) # higher tolerance for variance | |
| # Count basic punctuation | |
| punctuation_score = min(1.0, (sum(1 for c in text if c in '.,!?') / len(text)) * 2) # amplify good punctuation | |
| return min(1.0, (length_score * 0.7 + punctuation_score * 0.3)) | |
| def _calculate_wpm(self, audio_duration: float, transcribed_text: str) -> float: | |
| """Calculate Words Per Minute""" | |
| word_count = len(transcribed_text.split()) | |
| minutes = audio_duration / 60 | |
| return word_count / minutes if minutes > 0 else 0 | |
| def _get_assessment_prompt(self) -> PromptTemplate: | |
| """Create a prompt template for LLM assessment""" | |
| template = '''You are an interview evaluation assistant. Based on the given metrics—fluency, confidence, grammar, tone, and pauses—generate a brief, constructive assessment of the candidate's spoken response. Use a tone that is supportive and human-like, suitable for hiring and recruitment purposes. | |
| Audio Analysis Metrics: | |
| - Fluency Score: {fluency_score} | |
| - Grammar Score: {grammar_score} | |
| - Confidence Score: {confidence_score} | |
| - Clarity Score: {clarity_score} | |
| - Pronunciation Score: {pronunciation_score} | |
| - Words Per Minute: {wpm_rate} | |
| - Filler Words Count: {filler_words_count} | |
| - Speaking Duration: {speaking_duration} | |
| - Number of Pauses: {pauses_count} | |
| Transcribed Response: | |
| "{transcribed_text}" | |
| Job Role: {job_role} | |
| Please provide a comprehensive assessment including: | |
| 1. Overall communication effectiveness | |
| 2. Areas of strength | |
| 3. Areas for improvement | |
| 4. Interview performance score (0-100) | |
| 5. Hiring recommendation (recommend/consider/reject) | |
| Always mention: | |
| 1. Strengths: Start by highlighting what the candidate did well. | |
| 2. Areas for improvement: Be gentle, offering actionable suggestions. | |
| 3. Overall impression: Summarize whether the delivery is interview-ready or needs polishing. | |
| Be encouraging and avoid sounding robotic or overly critical. | |
| Format your response as a clear, concise paragraph that covers all these points. | |
| Focus on being direct and actionable in your assessment.''' | |
| return PromptTemplate( | |
| input_variables=[ | |
| "fluency_score", "grammar_score", "confidence_score", | |
| "clarity_score", "pronunciation_score", | |
| "wpm_rate", "filler_words_count", "speaking_duration", | |
| "pauses_count", "transcribed_text", "job_role" | |
| ], | |
| template=template | |
| ) | |
| def _get_llm_assessment(self, metrics: Dict, job_role: str) -> str: | |
| """Get LLM assessment of the audio analysis metrics""" | |
| try: | |
| llm = get_llm() | |
| prompt = self._get_assessment_prompt() | |
| # Prepare input variables | |
| input_vars = { | |
| **{k: metrics[k] for k in [ | |
| "fluency_score", "grammar_score", "confidence_score", | |
| "clarity_score", "pronunciation_score", | |
| "wpm_rate", "filler_words_count", "speaking_duration", | |
| "pauses_count", "transcribed_text" | |
| ]}, | |
| "job_role": job_role or "General" | |
| } | |
| # Get LLM response | |
| response = llm.invoke(prompt.format_prompt(**input_vars)) | |
| # Extract just the content from the response | |
| response_text = response.content.strip() | |
| if response_text.startswith("`"): | |
| response_text = response_text.strip("`") | |
| if response_text.startswith("json"): | |
| response_text = response_text[4:] | |
| return response_text.strip() | |
| except Exception as e: | |
| print(f"Error getting LLM assessment: {str(e)}") | |
| # Return a basic assessment based on metrics | |
| return f"""Basic analysis shows {metrics['speaking_duration']:.1f} seconds of speech with fluency score of {metrics['fluency_score']:.2f}. | |
| Speech clarity is {metrics['clarity_score']:.2f} and technical accuracy is {metrics['technical_accuracy']:.2f}. | |
| The candidate shows potential but could provide more technical details. | |
| Interview score: {int(metrics['confidence_score'] * 100)}/100. | |
| Recommendation: Consider for further evaluation.""" | |
| def analyze_audio(self, audio_data: bytes, job_role: str = None) -> Dict: | |
| """Main method to analyze audio and return a simplified response with transcription and assessment""" | |
| try: | |
| # Extract audio features | |
| audio_array, sample_rate = self._extract_audio_features(audio_data) | |
| duration = len(audio_array) / sample_rate | |
| # Get transcription | |
| transcribed_text = self._transcribe_audio(audio_array, sample_rate) | |
| # Calculate various scores internally | |
| fluency_score = self._calculate_fluency_score(audio_array, sample_rate) | |
| grammar_score = self._get_grammar_score(transcribed_text) | |
| confidence_score = self._calculate_confidence_score(audio_array, transcribed_text) | |
| tone_analysis = self._analyze_tone(transcribed_text) | |
| wpm = self._calculate_wpm(duration, transcribed_text) | |
| # Calculate clarity score based on audio features with adjusted parameters | |
| onset_env = librosa.onset.onset_strength(y=audio_array, sr=sample_rate) | |
| clarity_score = min(1.0, np.mean(onset_env) / 8) | |
| # Count pauses with more lenient parameters | |
| silence_intervals = librosa.effects.split(audio_array, top_db=35) | |
| pauses_count = len(silence_intervals) - 1 | |
| # Simplified filler words count with reduced list | |
| filler_words = ["um", "uh"] # Reduced list to only count significant fillers | |
| filler_count = sum(transcribed_text.lower().count(word) for word in filler_words) | |
| # Create metrics dictionary for LLM assessment | |
| metrics = { | |
| "fluency_score": float(fluency_score), | |
| "grammar_score": float(grammar_score), | |
| "confidence_score": float(confidence_score), | |
| "clarity_score": float(clarity_score), | |
| "pronunciation_score": float(clarity_score * 0.7 + fluency_score * 0.3), | |
| "transcribed_text": transcribed_text, | |
| "wpm_rate": float(wpm), | |
| "filler_words_count": filler_count, | |
| "speaking_duration": float(duration), | |
| "pauses_count": pauses_count | |
| } | |
| # Get LLM assessment | |
| assessment = self._get_llm_assessment(metrics, job_role) | |
| # Return simplified response | |
| return { | |
| "transcript": transcribed_text, | |
| "assessment": assessment | |
| } | |
| except Exception as e: | |
| print(f"Error in analyze_audio: {str(e)}") | |
| raise AudioAnalysisError(f"Failed to analyze audio: {str(e)}") |