Spaces:

ShadowTEM
/

Chatbot

Sleeping

uploaded the rest

8422173 verified over 1 year ago

7.88 kB

import os
import time
import json
import keyboard  # pip install keyboard==0.13.5
import numpy as np
import faiss
from datetime import datetime
from concurrent.futures import ThreadPoolExecutor, as_completed

# Import the AudioProcessor class from your STT module.
from stt import AudioProcessor
from llm import LLMProcessor
# Import vectorstore functions from vectorstore.py (located in app/)
from vectorstore import (
    update_vectorstore_from_pdf,
    save_vectorstore_with_timestamp,
    load_vectorstore,
    search_vectorstore
)
# Import TTS functions from the tts folder.
from tts import chunk_text_for_tts, synthesize_speech_guideline, play_audio


def combine_vectorstores(index1, chunks1, index2, chunks2):
    """
    Combine two FAISS indexes (both IndexFlatL2) by retrieving all embeddings,
    concatenating them, and building a new FAISS index.
    Also combines the two lists of text chunks.
    """
    embeddings1 = np.array([index1.reconstruct(i) for i in range(index1.ntotal)])
    embeddings2 = np.array([index2.reconstruct(i) for i in range(index2.ntotal)])
    combined_embeddings = np.concatenate([embeddings1, embeddings2], axis=0)
    combined_chunks = chunks1 + chunks2
    dimension = combined_embeddings.shape[1]
    combined_index = faiss.IndexFlatL2(dimension)
    combined_index.add(combined_embeddings)
    return combined_index, combined_chunks


def build_and_save_vectorstore():
    """
    Process two PDFs, combine their embeddings into one vectorstore,
    save the combined index with a timestamp, and return the index,
    chunks, and embedder used.
    """
    base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
    pdf_path1 = os.path.join(base_dir, "data", "PDF", "cancer_dictionary.pdf")
    pdf_path2 = os.path.join(base_dir, "data", "PDF", "Medical Dictionary.pdf")
    
    index1, chunks1, embedder = update_vectorstore_from_pdf(pdf_path1, chunk_size=600)
    index2, chunks2, _ = update_vectorstore_from_pdf(pdf_path2, chunk_size=600)
    
    combined_index, combined_chunks = combine_vectorstores(index1, chunks1, index2, chunks2)
    
    saved_index_path = save_vectorstore_with_timestamp(combined_index)
    print(f"Combined vectorstore saved at: {saved_index_path}")
    
    loaded_index = load_vectorstore(saved_index_path)
    return loaded_index, combined_chunks, embedder


def synthesize_audio_chunks(text, max_chars=300):
    """
    Breaks the given text into chunks and asynchronously synthesizes them.
    Yields the output file for each synthesized audio chunk.
    """
    if not text:
        raise ValueError("No text provided for TTS synthesis.")

    # Break the text into manageable chunks.
    chunks = chunk_text_for_tts(text, max_chars=max_chars)
    print("TTS Text chunks:")
    for i, chunk in enumerate(chunks, start=1):
        print(f"Chunk {i}: {chunk}\n")
    
    # Use a ThreadPoolExecutor to process the chunks concurrently.
    with ThreadPoolExecutor(max_workers=4) as executor:
        # Submit synthesis tasks and keep a mapping of future to its chunk index.
        future_to_index = {
            executor.submit(synthesize_speech_guideline, chunk, output_file=f"audio_chunk_{i}.wav"): i
            for i, chunk in enumerate(chunks, start=1)
        }
        
        # Yield audio files as soon as each future completes.
        for future in as_completed(future_to_index):
            index = future_to_index[future]
            try:
                audio_file = future.result()
                if audio_file is None:
                    print(f"Warning: Synthesis for chunk {index} returned None.")
                else:
                    print(f"Chunk {index} synthesized successfully: {audio_file}")
                    yield audio_file
            except Exception as exc:
                print(f"Error synthesizing chunk {index}: {exc}")

def synthesize_audio_chunks(text, max_chars=300):
    """
    Breaks the given text into chunks and asynchronously synthesizes them.
    Yields the output file for each synthesized audio chunk.
    """
    if not text:
        raise ValueError("No text provided for TTS synthesis.")
    chunks = chunk_text_for_tts(text, max_chars=max_chars)
    print("TTS Text chunks:")
    for i, chunk in enumerate(chunks, 1):
        print(f"Chunk {i}: {chunk}\n")
    
    with ThreadPoolExecutor(max_workers=4) as executor:
        # Submit synthesis tasks for each chunk
        futures = [executor.submit(synthesize_speech_guideline, chunk, output_file=f"audio_chunk_{i}.wav")
                    for i, chunk in enumerate(chunks)]
        # Yield audio files as soon as each synthesis task completes
        for future in futures:
            audio_file = future.result()
            yield audio_file


def main():
    print("Building vectorstore from PDFs...")
    vector_index, vector_chunks, embedder = build_and_save_vectorstore()
    
    processor = AudioProcessor()
    llm_processor = LLMProcessor()
    
    print("\nLooping process: Use push-to-talk to record and transcribe.")
    print("Press 's' at any time to stop the program.")
    Saved_response = ""
    
    while True:
        if keyboard.is_pressed('s'):
            print("Stop key pressed. Exiting program.")
            break
        
        print("\nReady to record. Press 'q' to start and 'q' again to stop recording.\n----------------\n")
        denoised_audio, fs = processor.record_and_denoise()
        
        if denoised_audio is None:
            print("No audio was recorded. Skipping transcription.")
        else:
            print("Transcribing audio...")
            try:
                transcription = processor.transcribe_audio(processor.default_output_file, language="en")
                print("Transcribed Text:\n", transcription)
                
                search_results = search_vectorstore(transcription, embedder, vector_index, vector_chunks, top_k=3)
                query = Saved_response + transcription
                for i, result in enumerate(search_results, 1):
                    query += f"\n{i}. {result}"
                
                try:
                    gemini_response = llm_processor.call_gemini_llm("gemini-2.0-flash", query)
                    print("Gemini response:", gemini_response)
                except Exception as e:
                    print("Gemini error:", e)
                    gemini_response = f"Error: {str(e)}"
                
                Saved_response += "transcription: " + transcription + "\n" + "gemini_response: " + gemini_response + "\n"
                processor.log_conversation(transcription, bot_text=gemini_response)
                
                # Synthesize and play TTS output in a pipelined fashion.
                print("Synthesizing and playing LLM response via TTS...")
                print()
                audio_generator = synthesize_audio_chunks(gemini_response, max_chars=300)
                # Immediately get the first audio chunk and play it
                try:
                    first_audio = next(audio_generator)
                    play_audio(first_audio)
                except StopIteration:
                    print("No audio chunks generated.")
                
                # Now play the rest of the chunks as soon as they are ready.
                for audio_file in audio_generator:
                    play_audio(audio_file)
                
            except Exception as e:
                print("Error during transcription:", e)
        
        print("Iteration complete. Waiting 5 seconds before next recording...")
        time.sleep(5)
    
    print("Program terminated.")



if __name__ == "__main__":
    main()

	import os
	import time
	import json
	import keyboard # pip install keyboard==0.13.5
	import numpy as np
	import faiss
	from datetime import datetime
	from concurrent.futures import ThreadPoolExecutor, as_completed

	# Import the AudioProcessor class from your STT module.
	from stt import AudioProcessor
	from llm import LLMProcessor
	# Import vectorstore functions from vectorstore.py (located in app/)
	from vectorstore import (
	update_vectorstore_from_pdf,
	save_vectorstore_with_timestamp,
	load_vectorstore,
	search_vectorstore
	)
	# Import TTS functions from the tts folder.
	from tts import chunk_text_for_tts, synthesize_speech_guideline, play_audio


	def combine_vectorstores(index1, chunks1, index2, chunks2):
	"""
	Combine two FAISS indexes (both IndexFlatL2) by retrieving all embeddings,
	concatenating them, and building a new FAISS index.
	Also combines the two lists of text chunks.
	"""
	embeddings1 = np.array([index1.reconstruct(i) for i in range(index1.ntotal)])
	embeddings2 = np.array([index2.reconstruct(i) for i in range(index2.ntotal)])
	combined_embeddings = np.concatenate([embeddings1, embeddings2], axis=0)
	combined_chunks = chunks1 + chunks2
	dimension = combined_embeddings.shape[1]
	combined_index = faiss.IndexFlatL2(dimension)
	combined_index.add(combined_embeddings)
	return combined_index, combined_chunks


	def build_and_save_vectorstore():
	"""
	Process two PDFs, combine their embeddings into one vectorstore,
	save the combined index with a timestamp, and return the index,
	chunks, and embedder used.
	"""
	base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
	pdf_path1 = os.path.join(base_dir, "data", "PDF", "cancer_dictionary.pdf")
	pdf_path2 = os.path.join(base_dir, "data", "PDF", "Medical Dictionary.pdf")

	index1, chunks1, embedder = update_vectorstore_from_pdf(pdf_path1, chunk_size=600)
	index2, chunks2, _ = update_vectorstore_from_pdf(pdf_path2, chunk_size=600)

	combined_index, combined_chunks = combine_vectorstores(index1, chunks1, index2, chunks2)

	saved_index_path = save_vectorstore_with_timestamp(combined_index)
	print(f"Combined vectorstore saved at: {saved_index_path}")

	loaded_index = load_vectorstore(saved_index_path)
	return loaded_index, combined_chunks, embedder


	def synthesize_audio_chunks(text, max_chars=300):
	"""
	Breaks the given text into chunks and asynchronously synthesizes them.
	Yields the output file for each synthesized audio chunk.
	"""
	if not text:
	raise ValueError("No text provided for TTS synthesis.")

	# Break the text into manageable chunks.
	chunks = chunk_text_for_tts(text, max_chars=max_chars)
	print("TTS Text chunks:")
	for i, chunk in enumerate(chunks, start=1):
	print(f"Chunk {i}: {chunk}\n")

	# Use a ThreadPoolExecutor to process the chunks concurrently.
	with ThreadPoolExecutor(max_workers=4) as executor:
	# Submit synthesis tasks and keep a mapping of future to its chunk index.
	future_to_index = {
	executor.submit(synthesize_speech_guideline, chunk, output_file=f"audio_chunk_{i}.wav"): i
	for i, chunk in enumerate(chunks, start=1)
	}

	# Yield audio files as soon as each future completes.
	for future in as_completed(future_to_index):
	index = future_to_index[future]
	try:
	audio_file = future.result()
	if audio_file is None:
	print(f"Warning: Synthesis for chunk {index} returned None.")
	else:
	print(f"Chunk {index} synthesized successfully: {audio_file}")
	yield audio_file
	except Exception as exc:
	print(f"Error synthesizing chunk {index}: {exc}")

	def synthesize_audio_chunks(text, max_chars=300):
	"""
	Breaks the given text into chunks and asynchronously synthesizes them.
	Yields the output file for each synthesized audio chunk.
	"""
	if not text:
	raise ValueError("No text provided for TTS synthesis.")
	chunks = chunk_text_for_tts(text, max_chars=max_chars)
	print("TTS Text chunks:")
	for i, chunk in enumerate(chunks, 1):
	print(f"Chunk {i}: {chunk}\n")

	with ThreadPoolExecutor(max_workers=4) as executor:
	# Submit synthesis tasks for each chunk
	futures = [executor.submit(synthesize_speech_guideline, chunk, output_file=f"audio_chunk_{i}.wav")
	for i, chunk in enumerate(chunks)]
	# Yield audio files as soon as each synthesis task completes
	for future in futures:
	audio_file = future.result()
	yield audio_file


	def main():
	print("Building vectorstore from PDFs...")
	vector_index, vector_chunks, embedder = build_and_save_vectorstore()

	processor = AudioProcessor()
	llm_processor = LLMProcessor()

	print("\nLooping process: Use push-to-talk to record and transcribe.")
	print("Press 's' at any time to stop the program.")
	Saved_response = ""

	while True:
	if keyboard.is_pressed('s'):
	print("Stop key pressed. Exiting program.")
	break

	print("\nReady to record. Press 'q' to start and 'q' again to stop recording.\n----------------\n")
	denoised_audio, fs = processor.record_and_denoise()

	if denoised_audio is None:
	print("No audio was recorded. Skipping transcription.")
	else:
	print("Transcribing audio...")
	try:
	transcription = processor.transcribe_audio(processor.default_output_file, language="en")
	print("Transcribed Text:\n", transcription)

	search_results = search_vectorstore(transcription, embedder, vector_index, vector_chunks, top_k=3)
	query = Saved_response + transcription
	for i, result in enumerate(search_results, 1):
	query += f"\n{i}. {result}"

	try:
	gemini_response = llm_processor.call_gemini_llm("gemini-2.0-flash", query)
	print("Gemini response:", gemini_response)
	except Exception as e:
	print("Gemini error:", e)
	gemini_response = f"Error: {str(e)}"

	Saved_response += "transcription: " + transcription + "\n" + "gemini_response: " + gemini_response + "\n"
	processor.log_conversation(transcription, bot_text=gemini_response)

	# Synthesize and play TTS output in a pipelined fashion.
	print("Synthesizing and playing LLM response via TTS...")
	print()
	audio_generator = synthesize_audio_chunks(gemini_response, max_chars=300)
	# Immediately get the first audio chunk and play it
	try:
	first_audio = next(audio_generator)
	play_audio(first_audio)
	except StopIteration:
	print("No audio chunks generated.")

	# Now play the rest of the chunks as soon as they are ready.
	for audio_file in audio_generator:
	play_audio(audio_file)

	except Exception as e:
	print("Error during transcription:", e)

	print("Iteration complete. Waiting 5 seconds before next recording...")
	time.sleep(5)

	print("Program terminated.")



	if __name__ == "__main__":
	main()