Spaces:
Runtime error
Runtime error
| from langchain.text_splitter import RecursiveCharacterTextSplitter | |
| from langchain.embeddings import HuggingFaceEmbeddings | |
| from langchain.vectorstores import FAISS | |
| from langchain.chains import ConversationalRetrievalChain | |
| from langchain.llms import HuggingFacePipeline | |
| from langchain.memory import ConversationBufferMemory | |
| from langchain_community.retrievers import BM25Retriever | |
| from langchain.retrievers import EnsembleRetriever | |
| import pandas as pd | |
| df = pd.read_csv('NLP.csv') | |
| corpus = df['text'][:300] | |
| #Chunking | |
| splitter = RecursiveCharacterTextSplitter(chunk_size=200,chunk_overlap = 10) | |
| texts = sum([splitter.split_text(doc) for doc in corpus], []) | |
| # Embeddings | |
| embeddings = HuggingFaceEmbeddings(model_name='all-mpnet-base-v2') | |
| # Indexing | |
| db = FAISS.from_texts(texts,embeddings) | |
| retriever = db.as_retriever(search_kwargs={'k':5}) | |
| # BM25 | |
| bm25 = BM25Retriever.from_texts(texts) | |
| bm25.k =5 | |
| # Hy_brid retriever | |
| hybrid_retriever = EnsembleRetriever( | |
| retrievers = [retriever,bm25], | |
| weights = [0.7,0.3] | |
| ) | |
| # Model | |
| llm = HuggingFacePipeline.from_model_id(model_id='google/flan-t5-large',task='text2text-generation') | |
| # Memory | |
| memory = ConversationBufferMemory(memory_key='chat_history',return_messages=True) | |
| # Combine previous steps | |
| qa = ConversationalRetrievalChain.from_llm(llm=llm,retriever=hybrid_retriever,memory=memory) | |
| def ans_ques(ques): | |
| result = qa({'question':ques}) | |
| return result['answer'] | |
| import gradio as gr | |
| demo = gr.Interface(ans_ques,inputs='text',outputs='text') | |
| demo.launch() | |