#import library from langchain.embeddings.openai import OpenAIEmbeddings from langchain.vectorstores import Chroma from langchain.text_splitter import CharacterTextSplitter from langchain.chains import ConversationalRetrievalChain import customnotiondbload as custom_notion from getpass import getpass NOTION_TOKEN ="secret_89vVgMQ1jAHNrbWWjAm0g2h9oEPZafSylgqEHmDvHhk" DATABASE_ID = "2bd913bed8374780854c776954cc9001" import os os.environ[ "OPENAI_API_KEY"] = "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" from langchain.chat_models import ChatOpenAI llm = ChatOpenAI(temperature=0, model_name="gpt-3.5-turbo") #Load data from langchain.document_loaders import DirectoryLoader # pdf_loader = DirectoryLoader('./Reports/', glob="**/*.pdf") # txt_loader = DirectoryLoader('./Reports/', glob="**/*.txt") # word_loader = DirectoryLoader('./Reports/', glob="**/*.docx") # html_loader = DirectoryLoader('./Reports/', glob="**/*.html") # lzone_loader= DirectoryLoader('./Reports/', glob="**/*.html") # csv_loader = DirectoryLoader('./Reports/', glob="**/*.csv") notion_loader = custom_notion.NotionDBLoader( integration_token=NOTION_TOKEN, database_id=DATABASE_ID, request_timeout_sec=60, # optional, defaults to 10 ) loaders = [notion_loader] documents = [] for loader in loaders: documents.extend(loader.load()) print(f"Total number of documents: {documents}") print(f"Total number of documents: {len(documents)}") #Chun data, Embeddings and save to VectorStore from langchain.vectorstores.utils import filter_complex_metadata text_splitter = CharacterTextSplitter(chunk_size=2000, chunk_overlap=200) # Filter out documents with complex metadata filtered_documents = [] for document in documents: print(document.metadata.values()) # print(all(value is None or isinstance(value, (str, int, float, bool)) for value in document.metadata.values())) if all(value is None or isinstance(value, (str, int, float, bool)) for value in document.metadata.values()): filtered_documents.append(document) embeddings = OpenAIEmbeddings() vectorstore = Chroma.from_documents(filtered_documents, embeddings) #Calling the Langchain's QA chain with Chat History qa = ConversationalRetrievalChain.from_llm(llm, vectorstore.as_retriever(), return_source_documents=True) #Gradio Chat UI import gradio as gr # def greet(name): # return "Hello " + name + "!" # demo = gr.Interface( # fn=greet, # inputs="text", # outputs="text", # css="footer{display:none !important}" # ) # demo.launch() # Define chat interface with gr.Blocks() as demo: chatbot = gr.Chatbot() msg = gr.Textbox() clear = gr.Button("Clear") chat_history = [] def user(query, chat_history): print("User query:", query) print("Chat history:", chat_history) # Convert chat history to list of tuples chat_history_tuples = [] for message in chat_history: chat_history_tuples.append((message[0], message[1])) # Get result from QA chain result = qa({"question": query, "chat_history": chat_history_tuples}) # Append user message and response to chat history chat_history.append((query, result["answer"])) print("Updated chat history:", chat_history) return gr.update(value=""), chat_history msg.submit(user, [msg, chatbot], [msg, chatbot], queue=False) clear.click(lambda: None, None, chatbot, queue=False) if __name__ == "__main__": demo.launch(share=True)