Spaces:

ekatra
/

mobius-v2

Paused

ramshashaikh

fixed index out of range error

135090c over 3 years ago

4.78 kB

import operations as op
import textwrap
# from dotenv import load_dotenv
import streamlit as st
import os
import nltk
from nltk.tokenize import sent_tokenize
import openai
nltk.download('punkt')


openai.organization = os.environ['org']
openai.api_key = os.environ['api_key']


st.title("Mobius")

# st.write("AI Powered Smart Search System")

st.markdown("""
<style>
.big-font {
    font-size:14px;
}
</style>
""", unsafe_allow_html=True)

st.markdown("""<p class="big-font">Mobius is the perfect way for companies to easily organize, access and leverage their data. With Mobius, companies can quickly and intuitively search their data to answer any question, while still staying organized and on track. Mobius' powerful semantic search capabilities cut through all the noise and access only the data that's necessary to get the answers companies need. With Mobius, companies can be smarter and more efficient in the way they work with data.</p>""", unsafe_allow_html=True)


# st.subheader(
# "With Mobius, companies can be smarter and more efficient in the way they work with data.")

top_match_sentences = []


@ st.cache_data
def process_pdf_data(uploaded_files):
    """
    The function accepts an uploaded PDF file as input and then proceeds to extract, preprocess, and vectorize the text. It ultimately returns a list of filtered sentences and their respective embeddings.
    Parameters:
    uploaded_files (list): List of uploaded files
    Returns:
    filt1_list (list): List of filtered sentences
    embeddings (list): List of embeddings of the sentences
    """
    filt1_list = []
    embeddings = []

    text_ext = []
    for i in uploaded_files:
        if i.type == "application/pdf":
            # Reading the pdf file and extracting the text
            text_ext += op.read_pdf(i)

    # Applying sent_tokenize to the text and storing the result in a list
    sent_toks = []
    for i in text_ext:
        sent_toks.append(sent_tokenize(i))

    concat_list = [j for i in sent_toks for j in i]

    # Removing the new line characters from the list
    for i in concat_list:
        a = (i.replace('\n', ' '))
        filt1_list.append(a)

    # Creating embeddings for the sentences
    embeddings = op.create_content_embeddings(filt1_list)
    return filt1_list, embeddings


# Streamlit code to upload files
uploaded_files = st.file_uploader(
    "Upload files - ", accept_multiple_files=True, type=['pdf'])

if st.button("Process!"):
    if len(uploaded_files) != 0:
        # Calling the function process_pdf_data to process the uploaded files
        filt1_list, embeddings = process_pdf_data(uploaded_files)
        st.write("Process Completed")

    else:
        st.warning("Please upload a PDF file.")

# Streamlit code to take user input after vectorization of the documents
query = st.text_input('Ask me anything!', placeholder='Type.....')
try:
    if st.button("Confirm!"):

        cached_data = process_pdf_data(uploaded_files)
        filt1_list = cached_data[0]
        embeddings = cached_data[1]

        # Creating embeddings for the query
        query_embedding = op.create_query_embeddings(query)

        # Calculating cosine similarity between the query and the sentences
        cosine_lis = op.calculate_cosine(
            query_embedding, embeddings, filt1_list)

        # Fetching the top 15 sentences with the highest cosine similarity
        indexes_final = op.fetch_top_rank_ans(cosine_lis, 15)

        # Fetching the most relevant sentence from the top 15 sentences, and providing it as the context to the GPT-3 model
        most_relevant = op.fetch_most_relevant(
            indexes_final, filt1_list, cosine_lis, query)

        # Calling the GPT-3 model to generate the answer
        # COMPLETIONS_API_PARAMS = {
        #     "temperature": 0.0,
        #     "max_tokens": 300,
        #     "model": "text-davinci-003",
        # }
        response = openai.ChatCompletion.create(
            model="gpt-3.5-turbo",
            messages=[
                {"role": "system", "content": "You are an educational assistant. Answer the questions with detail and in 200-300 characters only"},
                {"role": "user", "content": most_relevant},

            ]
        )
        # print("\n\n", textwrap.fill(
        #     response["choices"][0]["text"].strip(" \n")))

        # Displaying the answer to the user
        # print(response['choices'][0].message.content)
        st.write(textwrap.fill(response['choices'][0].message.content))
        # import streamlit_scrollable_textbox as stx

        # long_text = response['choices'][0].message.content
        # stx.scrollableTextbox(long_text, height=300)

except Exception as e:
    print(e)
    st.warning("Something went wrong. Please try again.")

	import operations as op
	import textwrap
	# from dotenv import load_dotenv
	import streamlit as st
	import os
	import nltk
	from nltk.tokenize import sent_tokenize
	import openai
	nltk.download('punkt')


	openai.organization = os.environ['org']
	openai.api_key = os.environ['api_key']


	st.title("Mobius")

	# st.write("AI Powered Smart Search System")

	st.markdown("""
	<style>
	.big-font {
	font-size:14px;
	}
	</style>
	""", unsafe_allow_html=True)

	st.markdown("""<p class="big-font">Mobius is the perfect way for companies to easily organize, access and leverage their data. With Mobius, companies can quickly and intuitively search their data to answer any question, while still staying organized and on track. Mobius' powerful semantic search capabilities cut through all the noise and access only the data that's necessary to get the answers companies need. With Mobius, companies can be smarter and more efficient in the way they work with data.</p>""", unsafe_allow_html=True)


	# st.subheader(
	# "With Mobius, companies can be smarter and more efficient in the way they work with data.")

	top_match_sentences = []


	@ st.cache_data
	def process_pdf_data(uploaded_files):
	"""
	The function accepts an uploaded PDF file as input and then proceeds to extract, preprocess, and vectorize the text. It ultimately returns a list of filtered sentences and their respective embeddings.

	Parameters:
	uploaded_files (list): List of uploaded files

	Returns:
	filt1_list (list): List of filtered sentences
	embeddings (list): List of embeddings of the sentences

	"""
	filt1_list = []
	embeddings = []

	text_ext = []
	for i in uploaded_files:
	if i.type == "application/pdf":
	# Reading the pdf file and extracting the text
	text_ext += op.read_pdf(i)

	# Applying sent_tokenize to the text and storing the result in a list
	sent_toks = []
	for i in text_ext:
	sent_toks.append(sent_tokenize(i))

	concat_list = [j for i in sent_toks for j in i]

	# Removing the new line characters from the list
	for i in concat_list:
	a = (i.replace('\n', ' '))
	filt1_list.append(a)

	# Creating embeddings for the sentences
	embeddings = op.create_content_embeddings(filt1_list)
	return filt1_list, embeddings


	# Streamlit code to upload files
	uploaded_files = st.file_uploader(
	"Upload files - ", accept_multiple_files=True, type=['pdf'])

	if st.button("Process!"):
	if len(uploaded_files) != 0:
	# Calling the function process_pdf_data to process the uploaded files
	filt1_list, embeddings = process_pdf_data(uploaded_files)
	st.write("Process Completed")

	else:
	st.warning("Please upload a PDF file.")

	# Streamlit code to take user input after vectorization of the documents
	query = st.text_input('Ask me anything!', placeholder='Type.....')
	try:
	if st.button("Confirm!"):

	cached_data = process_pdf_data(uploaded_files)
	filt1_list = cached_data[0]
	embeddings = cached_data[1]

	# Creating embeddings for the query
	query_embedding = op.create_query_embeddings(query)

	# Calculating cosine similarity between the query and the sentences
	cosine_lis = op.calculate_cosine(
	query_embedding, embeddings, filt1_list)

	# Fetching the top 15 sentences with the highest cosine similarity
	indexes_final = op.fetch_top_rank_ans(cosine_lis, 15)

	# Fetching the most relevant sentence from the top 15 sentences, and providing it as the context to the GPT-3 model
	most_relevant = op.fetch_most_relevant(
	indexes_final, filt1_list, cosine_lis, query)

	# Calling the GPT-3 model to generate the answer
	# COMPLETIONS_API_PARAMS = {
	# "temperature": 0.0,
	# "max_tokens": 300,
	# "model": "text-davinci-003",
	# }
	response = openai.ChatCompletion.create(
	model="gpt-3.5-turbo",
	messages=[
	{"role": "system", "content": "You are an educational assistant. Answer the questions with detail and in 200-300 characters only"},
	{"role": "user", "content": most_relevant},

	]
	)
	# print("\n\n", textwrap.fill(
	# response["choices"][0]["text"].strip(" \n")))

	# Displaying the answer to the user
	# print(response['choices'][0].message.content)
	st.write(textwrap.fill(response['choices'][0].message.content))
	# import streamlit_scrollable_textbox as stx

	# long_text = response['choices'][0].message.content
	# stx.scrollableTextbox(long_text, height=300)

	except Exception as e:
	print(e)
	st.warning("Something went wrong. Please try again.")