Create app.py
Browse files
app.py
ADDED
|
@@ -0,0 +1,174 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import itertools
|
| 2 |
+
import os
|
| 3 |
+
import gradio as gr
|
| 4 |
+
import openai
|
| 5 |
+
from langchain.chains import RetrievalQA
|
| 6 |
+
from langchain.document_loaders.directory import DirectoryLoader
|
| 7 |
+
from langchain.memory import ConversationBufferWindowMemory
|
| 8 |
+
from langchain.text_splitter import CharacterTextSplitter
|
| 9 |
+
import textwrap
|
| 10 |
+
from langchain.embeddings import OpenAIEmbeddings
|
| 11 |
+
from langchain.vectorstores import Chroma
|
| 12 |
+
from langchain_community.document_loaders import UnstructuredURLLoader
|
| 13 |
+
from langchain_core.callbacks import StreamingStdOutCallbackHandler
|
| 14 |
+
from langchain.prompts import PromptTemplate
|
| 15 |
+
from langchain.chat_models import ChatOpenAI
|
| 16 |
+
from pdfminer.high_level import extract_text
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
os.environ["HUGGINGFACEHUB_API_TOKEN"] = os.getenv('HUGGINGFACEHUB_API_TOKEN')
|
| 20 |
+
openai.api_key = os.getenv('OPENAI_API_KEY')
|
| 21 |
+
#loader = DirectoryLoader('documents/')
|
| 22 |
+
docs_dir = 'documents/'
|
| 23 |
+
txts_dir = docs_dir + 'txt/'
|
| 24 |
+
converted_txts_dir = docs_dir + 'converted_txt/'
|
| 25 |
+
# pdfs_dir = docs_dir + 'pdf/'
|
| 26 |
+
# txt_documents = [os.path.splitext(doc)[0] for doc in list(os.listdir(converted_txts_dir))]
|
| 27 |
+
# pdf_documents = [os.path.splitext(doc)[0] for doc in list(os.listdir(pdfs_dir))]
|
| 28 |
+
# for pdf_doc in pdf_documents:
|
| 29 |
+
# if pdf_doc not in txt_documents:
|
| 30 |
+
# print("Converting " + pdf_doc + ' pdf to txt')
|
| 31 |
+
# text = extract_text(pdfs_dir + pdf_doc + '.pdf')
|
| 32 |
+
# with open(converted_txts_dir + pdf_doc + '.txt', 'w') as f:
|
| 33 |
+
# f.write(text)
|
| 34 |
+
# intents_df = pd.read_csv(docs_dir + 'intents.csv')
|
| 35 |
+
# intents_df.sort_values(by=['intent', 'response'], inplace=True)
|
| 36 |
+
# previous_row = None
|
| 37 |
+
# for index, row in intents_df.iterrows():
|
| 38 |
+
# if previous_row is not None and previous_row['intent'] == row['intent']:
|
| 39 |
+
# intent_index = intent_index + 1
|
| 40 |
+
# else:
|
| 41 |
+
# intent_index = 0
|
| 42 |
+
# with open(f'{docs_dir}intents/{row["intent"]}_{intent_index}.txt', 'w') as f:
|
| 43 |
+
# f.write(row['response'])
|
| 44 |
+
# previous_row = row
|
| 45 |
+
docs_dir = 'documents/'
|
| 46 |
+
txts_dir = docs_dir + 'txt/'
|
| 47 |
+
# converted_txts_dir = docs_dir + 'converted_txt/'
|
| 48 |
+
# pdfs_dir = docs_dir + 'pdf/'
|
| 49 |
+
# txt_documents = [os.path.splitext(doc)[0] for doc in list(os.listdir(converted_txts_dir))]
|
| 50 |
+
# pdf_documents = [os.path.splitext(doc)[0] for doc in list(os.listdir(pdfs_dir))]
|
| 51 |
+
# for pdf_doc in pdf_documents:
|
| 52 |
+
# if pdf_doc not in txt_documents:
|
| 53 |
+
# print("Converting " + pdf_doc + ' pdf to txt')
|
| 54 |
+
# text = extract_text(pdfs_dir + pdf_doc + '.pdf')
|
| 55 |
+
# with open(converted_txts_dir + pdf_doc + '.txt', 'w') as f:
|
| 56 |
+
# f.write(text)
|
| 57 |
+
# intents_df = pd.read_csv(docs_dir + 'intents.csv')
|
| 58 |
+
# intents_df.sort_values(by=['intent', 'response'], inplace=True)
|
| 59 |
+
# previous_row = None
|
| 60 |
+
# for index, row in intents_df.iterrows():
|
| 61 |
+
# if previous_row is not None and previous_row['intent'] == row['intent']:
|
| 62 |
+
# intent_index = intent_index + 1
|
| 63 |
+
# else:
|
| 64 |
+
# intent_index = 0
|
| 65 |
+
# with open(f'{docs_dir}intents/{row["intent"]}_{intent_index}.txt', 'w') as f:
|
| 66 |
+
# f.write(row['response'])
|
| 67 |
+
# previous_row = row
|
| 68 |
+
loader = DirectoryLoader(txts_dir)
|
| 69 |
+
txt_documents = loader.load()
|
| 70 |
+
loader = UnstructuredURLLoader(urls=['https://stewartdavidpi.com/category/una-sociedad-magica/',
|
| 71 |
+
'https://stewartdavidpi.com/category/hijos-de-kirves/',
|
| 72 |
+
'https://stewartdavidpi.com/category/el-poder-tormerio/',
|
| 73 |
+
'https://stewartdavidpi.com/category/el-senor-de-gon-dhul/',
|
| 74 |
+
'https://stewartdavidpi.com/category/la-evolucion-tadur/',
|
| 75 |
+
'https://stewartdavidpi.com/sobre-mi/',
|
| 76 |
+
'https://stewartdavidpi.com/2023/12/01/sobre-las-ilustraciones-del-blog/',
|
| 77 |
+
'https://stewartdavidpi.com/2023/11/17/mapa-de-inggrim/',
|
| 78 |
+
'https://stewartdavidpi.com/2023/11/10/desarrollo-creativo-del-mapa-de-inggrim-parte-1-2/',
|
| 79 |
+
'https://stewartdavidpi.com/2023/11/17/desarrollo-creativo-del-mapa-de-inggrim-2-2/',
|
| 80 |
+
'https://stewartdavidpi.com/2023/11/03/tenemos-banda-sonora/'])
|
| 81 |
+
url_documents = loader.load()
|
| 82 |
+
def wrap_text_preserve_newlines(text, width=110):
|
| 83 |
+
# Split the input text into lines based on newline characters
|
| 84 |
+
lines = text.split('\n')
|
| 85 |
+
# Wrap each line individually
|
| 86 |
+
wrapped_lines = [textwrap.fill(line, width=width) for line in lines]
|
| 87 |
+
# Join the wrapped lines back together using newline characters
|
| 88 |
+
wrapped_text = '\n'.join(wrapped_lines)
|
| 89 |
+
return wrapped_text
|
| 90 |
+
|
| 91 |
+
max_length = 1000
|
| 92 |
+
text_splitter = CharacterTextSplitter(chunk_size=max_length, chunk_overlap=10)
|
| 93 |
+
docs = text_splitter.split_documents(itertools.chain(txt_documents, url_documents))
|
| 94 |
+
|
| 95 |
+
model_name = "decapoda-research/llama-7b-hf"
|
| 96 |
+
model_name = "gpt-3.5-turbo"
|
| 97 |
+
#model_name = "gpt2"
|
| 98 |
+
#tokenizer = GPT2Tokenizer.from_pretrained(model_name)
|
| 99 |
+
#tokenizer = tiktoken.get_encoding("gpt2")
|
| 100 |
+
#assert tokenizer.decode(tokenizer.encode("hola mundo")) == "hola mundo"
|
| 101 |
+
# tokenizer._pad_token = tokenizer.eos_token
|
| 102 |
+
# tokenizer._pad_token_id = 50256
|
| 103 |
+
#model = GPT2LMHeadModel.from_pretrained(model_name)
|
| 104 |
+
embeddings = OpenAIEmbeddings()
|
| 105 |
+
vector_store = Chroma.from_documents(docs, embeddings)
|
| 106 |
+
model_kwargs = {"max_length": max_length, "pad_token_id": 50256, "pad_token": "[PAD]", "padding": 'longest'}
|
| 107 |
+
# llm = HuggingFaceHub(repo_id=model_name, model_kwargs={"temperature": 0.1, "max_length": 500,
|
| 108 |
+
# "pad_token_id": 50256, "pad_token": "[PAD]",
|
| 109 |
+
# "padding": 'longest' #for open end generation
|
| 110 |
+
# })
|
| 111 |
+
llm = ChatOpenAI(streaming=True, model_name=model_name, temperature=0.9)
|
| 112 |
+
prompt_template = """You are going to
|
| 113 |
+
receive a context with information about a spanish writer and his sets of novels.
|
| 114 |
+
You can only answer things based on the question and a context to answer it.
|
| 115 |
+
Context: {context}
|
| 116 |
+
Question: {question}"""
|
| 117 |
+
#Memory: {chat_history}"""
|
| 118 |
+
|
| 119 |
+
PROMPT = PromptTemplate(
|
| 120 |
+
template=prompt_template, input_variables=["context", "question"]
|
| 121 |
+
)
|
| 122 |
+
chain_type_kwargs = {"prompt": PROMPT}
|
| 123 |
+
memory = ConversationBufferWindowMemory(k=4, memory_key="chat_history", input_key="question", output_key='answer',
|
| 124 |
+
return_messages=True)
|
| 125 |
+
|
| 126 |
+
retriever = vector_store.as_retriever()
|
| 127 |
+
# memory = VectorStoreRetrieverMemory(
|
| 128 |
+
# memory_key="chat_history",
|
| 129 |
+
# return_docs=False,
|
| 130 |
+
# retriever=retriever,
|
| 131 |
+
# return_messages=True,
|
| 132 |
+
# )
|
| 133 |
+
qa = RetrievalQA.from_chain_type(llm=llm, chain_type="stuff", retriever=retriever,
|
| 134 |
+
chain_type_kwargs=chain_type_kwargs, return_source_documents=True)
|
| 135 |
+
#memory=memory)
|
| 136 |
+
# qa = ConversationalRetrievalChain.from_llm(
|
| 137 |
+
# llm=llm, retriever=retriever, memory=memory,
|
| 138 |
+
# return_source_documents=True,
|
| 139 |
+
# combine_docs_chain_kwargs=chain_type_kwargs)
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
# while True:
|
| 143 |
+
# query = input("Question: ")
|
| 144 |
+
# #docs = vector_store.similarity_search(query)
|
| 145 |
+
# #print(wrap_text_preserve_newlines(str(docs[0].page_content)))
|
| 146 |
+
# response = qa({"query": query})
|
| 147 |
+
# print(f"Answer: {response['result']}")
|
| 148 |
+
# print(f"Sources: {response['source_documents']}")
|
| 149 |
+
|
| 150 |
+
def chat(message, history):
|
| 151 |
+
result = qa({"query": message})
|
| 152 |
+
sources_list = []
|
| 153 |
+
for item in result['source_documents']:
|
| 154 |
+
if 'http' in item.metadata['source']:
|
| 155 |
+
sources_list = sources_list + [item.metadata['source']]
|
| 156 |
+
sources_list = list(set(sources_list))
|
| 157 |
+
response = result['result']
|
| 158 |
+
if len(sources_list) > 0:
|
| 159 |
+
response = response + '\n\nSources:\n' + '\n'.join(sources_list)
|
| 160 |
+
yield response
|
| 161 |
+
# print(f"Answer: {response['result']}")
|
| 162 |
+
# print(f"Sources: {response['source_documents']}")
|
| 163 |
+
|
| 164 |
+
gr.ChatInterface(chat, chatbot=gr.Chatbot(),
|
| 165 |
+
textbox=gr.Textbox(placeholder="Pregúntame lo que quieras sobre INGGRIM!", container=False, scale=7),
|
| 166 |
+
title="Todo sobre la novela río: INGGRIM",
|
| 167 |
+
description="Pregúntame lo que quieras sobre INGGRIM!",
|
| 168 |
+
theme="soft",
|
| 169 |
+
examples=["¿Quién es Stewart David?", "¿Cuáles son las facciones presentes en INGGRIM?", "¿Cómo es el mapa del mundo de INGGRIM?"],
|
| 170 |
+
cache_examples=False,
|
| 171 |
+
retry_btn=None,
|
| 172 |
+
undo_btn="Borrar anterior",
|
| 173 |
+
clear_btn="Borrar").queue().launch()
|
| 174 |
+
|