File size: 8,409 Bytes
6f02abb 683d13b 6f02abb 4a89bd5 6f02abb 8f0ad76 6f02abb 94a6a58 6f02abb aaf26a2 6f02abb | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 | import itertools
import os
import gradio as gr
import openai
from langchain.chains import RetrievalQA
from langchain.document_loaders.directory import DirectoryLoader
from langchain.memory import ConversationBufferWindowMemory
from langchain.text_splitter import CharacterTextSplitter
import textwrap
from langchain.embeddings import OpenAIEmbeddings
from langchain.vectorstores import Chroma
from langchain_community.document_loaders import UnstructuredURLLoader
from langchain_core.callbacks import StreamingStdOutCallbackHandler
from langchain.prompts import PromptTemplate
from langchain.chat_models import ChatOpenAI
from pdfminer.high_level import extract_text
os.environ["HUGGINGFACEHUB_API_TOKEN"] = os.getenv('HUGGINGFACEHUB_API_TOKEN')
openai.api_key = os.getenv('OPENAI_API_KEY')
#loader = DirectoryLoader('documents/')
docs_dir = 'documents/'
txts_dir = docs_dir + 'txt/'
converted_txts_dir = docs_dir + 'converted_txt/'
# pdfs_dir = docs_dir + 'pdf/'
# txt_documents = [os.path.splitext(doc)[0] for doc in list(os.listdir(converted_txts_dir))]
# pdf_documents = [os.path.splitext(doc)[0] for doc in list(os.listdir(pdfs_dir))]
# for pdf_doc in pdf_documents:
# if pdf_doc not in txt_documents:
# print("Converting " + pdf_doc + ' pdf to txt')
# text = extract_text(pdfs_dir + pdf_doc + '.pdf')
# with open(converted_txts_dir + pdf_doc + '.txt', 'w') as f:
# f.write(text)
# intents_df = pd.read_csv(docs_dir + 'intents.csv')
# intents_df.sort_values(by=['intent', 'response'], inplace=True)
# previous_row = None
# for index, row in intents_df.iterrows():
# if previous_row is not None and previous_row['intent'] == row['intent']:
# intent_index = intent_index + 1
# else:
# intent_index = 0
# with open(f'{docs_dir}intents/{row["intent"]}_{intent_index}.txt', 'w') as f:
# f.write(row['response'])
# previous_row = row
docs_dir = 'documents/'
txts_dir = docs_dir + 'txt/'
# converted_txts_dir = docs_dir + 'converted_txt/'
# pdfs_dir = docs_dir + 'pdf/'
# txt_documents = [os.path.splitext(doc)[0] for doc in list(os.listdir(converted_txts_dir))]
# pdf_documents = [os.path.splitext(doc)[0] for doc in list(os.listdir(pdfs_dir))]
# for pdf_doc in pdf_documents:
# if pdf_doc not in txt_documents:
# print("Converting " + pdf_doc + ' pdf to txt')
# text = extract_text(pdfs_dir + pdf_doc + '.pdf')
# with open(converted_txts_dir + pdf_doc + '.txt', 'w') as f:
# f.write(text)
# intents_df = pd.read_csv(docs_dir + 'intents.csv')
# intents_df.sort_values(by=['intent', 'response'], inplace=True)
# previous_row = None
# for index, row in intents_df.iterrows():
# if previous_row is not None and previous_row['intent'] == row['intent']:
# intent_index = intent_index + 1
# else:
# intent_index = 0
# with open(f'{docs_dir}intents/{row["intent"]}_{intent_index}.txt', 'w') as f:
# f.write(row['response'])
# previous_row = row
#loader = DirectoryLoader(txts_dir)
#txt_documents = loader.load()
loader = UnstructuredURLLoader(urls=['https://stewartdavidpi.com/category/una-sociedad-magica/',
'https://stewartdavidpi.com/category/hijos-de-kirves/',
'https://stewartdavidpi.com/category/el-poder-tormerio/',
'https://stewartdavidpi.com/category/el-senor-de-gon-dhul/',
'https://stewartdavidpi.com/category/la-evolucion-tadur/',
'https://stewartdavidpi.com/sobre-mi/',
'https://stewartdavidpi.com/2023/12/01/sobre-las-ilustraciones-del-blog/',
'https://stewartdavidpi.com/2023/11/17/mapa-de-inggrim/',
'https://stewartdavidpi.com/2023/11/10/desarrollo-creativo-del-mapa-de-inggrim-parte-1-2/',
'https://stewartdavidpi.com/2023/11/17/desarrollo-creativo-del-mapa-de-inggrim-2-2/',
'https://stewartdavidpi.com/2023/11/03/tenemos-banda-sonora/'])
url_documents = loader.load()
def wrap_text_preserve_newlines(text, width=110):
# Split the input text into lines based on newline characters
lines = text.split('\n')
# Wrap each line individually
wrapped_lines = [textwrap.fill(line, width=width) for line in lines]
# Join the wrapped lines back together using newline characters
wrapped_text = '\n'.join(wrapped_lines)
return wrapped_text
max_length = 1000
text_splitter = CharacterTextSplitter(chunk_size=max_length, chunk_overlap=10)
docs = text_splitter.split_documents(url_documents)
model_name = "decapoda-research/llama-7b-hf"
model_name = "gpt-3.5-turbo"
#model_name = "gpt2"
#tokenizer = GPT2Tokenizer.from_pretrained(model_name)
#tokenizer = tiktoken.get_encoding("gpt2")
#assert tokenizer.decode(tokenizer.encode("hola mundo")) == "hola mundo"
# tokenizer._pad_token = tokenizer.eos_token
# tokenizer._pad_token_id = 50256
#model = GPT2LMHeadModel.from_pretrained(model_name)
embeddings = OpenAIEmbeddings()
vector_store = Chroma.from_documents(docs, embeddings)
model_kwargs = {"max_length": max_length, "pad_token_id": 50256, "pad_token": "[PAD]", "padding": 'longest'}
# llm = HuggingFaceHub(repo_id=model_name, model_kwargs={"temperature": 0.1, "max_length": 500,
# "pad_token_id": 50256, "pad_token": "[PAD]",
# "padding": 'longest' #for open end generation
# })
llm = ChatOpenAI(streaming=True, model_name=model_name, temperature=0.9)
prompt_template = """You are going to
receive a context with information about a spanish writer and his sets of novels.
You can only answer things based on the question and a context to answer it. You have to answer in the same language you are being asked.
Context: {context}
Question: {question}"""
#Memory: {chat_history}"""
PROMPT = PromptTemplate(
template=prompt_template, input_variables=["context", "question"]
)
chain_type_kwargs = {"prompt": PROMPT}
memory = ConversationBufferWindowMemory(k=4, memory_key="chat_history", input_key="question", output_key='answer',
return_messages=True)
retriever = vector_store.as_retriever()
# memory = VectorStoreRetrieverMemory(
# memory_key="chat_history",
# return_docs=False,
# retriever=retriever,
# return_messages=True,
# )
qa = RetrievalQA.from_chain_type(llm=llm, chain_type="stuff", retriever=retriever,
chain_type_kwargs=chain_type_kwargs, return_source_documents=True)
#memory=memory)
# qa = ConversationalRetrievalChain.from_llm(
# llm=llm, retriever=retriever, memory=memory,
# return_source_documents=True,
# combine_docs_chain_kwargs=chain_type_kwargs)
# while True:
# query = input("Question: ")
# #docs = vector_store.similarity_search(query)
# #print(wrap_text_preserve_newlines(str(docs[0].page_content)))
# response = qa({"query": query})
# print(f"Answer: {response['result']}")
# print(f"Sources: {response['source_documents']}")
def chat(message, history):
result = qa({"query": message})
sources_list = []
for item in result['source_documents']:
if 'http' in item.metadata['source']:
sources_list = sources_list + [item.metadata['source']]
sources_list = list(set(sources_list))
response = result['result']
if len(sources_list) > 0:
response = response + '\n\nFuentes:\n' + '\n'.join(sources_list)
yield response
# print(f"Answer: {response['result']}")
# print(f"Sources: {response['source_documents']}")
gr.ChatInterface(chat, chatbot=gr.Chatbot(),
textbox=gr.Textbox(placeholder="Pregúntame lo que quieras sobre INGGRIM!", container=False, scale=7),
title="Todo sobre la novela río: INGGRIM",
description="Pregúntame lo que quieras sobre INGGRIM!",
theme="soft",
examples=["¿Quién es Stewart David?", "¿Cómo se concibió el mapa de INGGRIM?"],
cache_examples=False,
retry_btn=None,
undo_btn="Borrar anterior",
clear_btn="Borrar").queue().launch()
|