Oriol Ganduxé Pregona commited on
Commit ·
9ed6379
1
Parent(s): 92eb375
Adding Tuist embeddings and checkbox to use those instead of loading them from openai.
Browse files- code_qa_app.py +73 -3
- tuist_cached_db/chroma-collections.parquet +3 -0
- tuist_cached_db/chroma-embeddings.parquet +3 -0
- tuist_cached_db/index/id_to_uuid_219bcda1-1619-4ee4-8db7-38b0df6259ed.pkl +3 -0
- tuist_cached_db/index/index_219bcda1-1619-4ee4-8db7-38b0df6259ed.bin +3 -0
- tuist_cached_db/index/index_metadata_219bcda1-1619-4ee4-8db7-38b0df6259ed.pkl +3 -0
- tuist_cached_db/index/uuid_to_id_219bcda1-1619-4ee4-8db7-38b0df6259ed.pkl +3 -0
code_qa_app.py
CHANGED
|
@@ -8,6 +8,9 @@ from langchain.indexes import VectorstoreIndexCreator
|
|
| 8 |
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
| 9 |
from langchain.embeddings import OpenAIEmbeddings
|
| 10 |
from langchain.vectorstores import Chroma
|
|
|
|
|
|
|
|
|
|
| 11 |
import panel as pn
|
| 12 |
|
| 13 |
# UI configuration
|
|
@@ -23,6 +26,8 @@ pn.state.template.param.update(
|
|
| 23 |
)
|
| 24 |
|
| 25 |
# User inputs
|
|
|
|
|
|
|
| 26 |
url_repo = pn.widgets.TextInput(
|
| 27 |
value="",
|
| 28 |
placeholder="Enter your GitHub repo URL here...",
|
|
@@ -63,6 +68,19 @@ select_chain_type = pn.widgets.RadioButtonGroup(
|
|
| 63 |
options=['stuff', 'map_reduce', "refine", "map_rerank"]
|
| 64 |
)
|
| 65 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 66 |
# Question answering function
|
| 67 |
def qa(url_repo, branch, query, chain_type, k):
|
| 68 |
# load data
|
|
@@ -99,6 +117,41 @@ def qa(url_repo, branch, query, chain_type, k):
|
|
| 99 |
print(result['result'])
|
| 100 |
return result
|
| 101 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 102 |
|
| 103 |
convos = [] # store all panel objects in a list
|
| 104 |
|
|
@@ -107,7 +160,12 @@ def qa_result(_):
|
|
| 107 |
|
| 108 |
prompt_text = prompt.value
|
| 109 |
if prompt_text:
|
| 110 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 111 |
convos.extend([
|
| 112 |
pn.Row(
|
| 113 |
pn.panel("\U0001F60A", width=10),
|
|
@@ -135,6 +193,18 @@ qa_interactive = pn.panel(
|
|
| 135 |
output = pn.WidgetBox('*Output will show up here:*', qa_interactive, width=630, scroll=True)
|
| 136 |
|
| 137 |
# Layout
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 138 |
widgets = pn.Row(
|
| 139 |
pn.Column(
|
| 140 |
prompt,
|
|
@@ -160,8 +230,8 @@ pn.Column(
|
|
| 160 |
3) Type a question and click "Run".\n
|
| 161 |
|
| 162 |
"""),
|
| 163 |
-
pn.Column(
|
|
|
|
| 164 |
output,
|
| 165 |
-
widgets
|
| 166 |
|
| 167 |
).servable()
|
|
|
|
| 8 |
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
| 9 |
from langchain.embeddings import OpenAIEmbeddings
|
| 10 |
from langchain.vectorstores import Chroma
|
| 11 |
+
from langchain.chat_models import ChatOpenAI
|
| 12 |
+
from typing import List
|
| 13 |
+
from langchain.vectorstores import Chroma
|
| 14 |
import panel as pn
|
| 15 |
|
| 16 |
# UI configuration
|
|
|
|
| 26 |
)
|
| 27 |
|
| 28 |
# User inputs
|
| 29 |
+
use_cached_repo = pn.widgets.Checkbox(name="Use cached Tuist repo instead")
|
| 30 |
+
|
| 31 |
url_repo = pn.widgets.TextInput(
|
| 32 |
value="",
|
| 33 |
placeholder="Enter your GitHub repo URL here...",
|
|
|
|
| 68 |
options=['stuff', 'map_reduce', "refine", "map_rerank"]
|
| 69 |
)
|
| 70 |
|
| 71 |
+
# class HardcodedRetriever(BaseRetriever):
|
| 72 |
+
# def __init__(self, documents: [Document]):
|
| 73 |
+
# self.documents = documents
|
| 74 |
+
|
| 75 |
+
# def get_relevant_documents(self, query: str) -> List[Document]:
|
| 76 |
+
# # Ignoring query
|
| 77 |
+
# return self.documents
|
| 78 |
+
|
| 79 |
+
# async def aget_relevant_documents(self, query: str) -> List[Document]:
|
| 80 |
+
# # Ignoring query
|
| 81 |
+
# return self.documents
|
| 82 |
+
|
| 83 |
+
|
| 84 |
# Question answering function
|
| 85 |
def qa(url_repo, branch, query, chain_type, k):
|
| 86 |
# load data
|
|
|
|
| 117 |
print(result['result'])
|
| 118 |
return result
|
| 119 |
|
| 120 |
+
# Question answering function
|
| 121 |
+
def qa_cached_tuist(query, chain_type, k):
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
# select which embeddings to use
|
| 125 |
+
embeddings = OpenAIEmbeddings()
|
| 126 |
+
|
| 127 |
+
# create the vectorstore to use as the index
|
| 128 |
+
persist_directory = "tuist_cached_db/"
|
| 129 |
+
db = Chroma(persist_directory=persist_directory, embedding_function=embeddings)
|
| 130 |
+
|
| 131 |
+
llm = ChatOpenAI(
|
| 132 |
+
model_name = "gpt-3.5-turbo",
|
| 133 |
+
temperature=0.7,
|
| 134 |
+
max_tokens=1000)
|
| 135 |
+
|
| 136 |
+
# You can test how an empty retriever makes our LLM ignorant about the context
|
| 137 |
+
# hardcodedRetriever = HardcodedRetriever([])
|
| 138 |
+
|
| 139 |
+
# qa = RetrievalQA.from_chain_type(llm=llm, chain_type="stuff", retriever=hardcodedRetriever)
|
| 140 |
+
|
| 141 |
+
# expose this index in a retriever interface
|
| 142 |
+
retriever = db.as_retriever(search_type="similarity", search_kwargs={"k": k})
|
| 143 |
+
|
| 144 |
+
# create a chain to answer questions
|
| 145 |
+
qa = RetrievalQA.from_chain_type(
|
| 146 |
+
llm=OpenAI(),
|
| 147 |
+
chain_type=chain_type,
|
| 148 |
+
retriever=retriever,
|
| 149 |
+
return_source_documents=True
|
| 150 |
+
)
|
| 151 |
+
|
| 152 |
+
result = qa({"query": query})
|
| 153 |
+
print(result['result'])
|
| 154 |
+
return result
|
| 155 |
|
| 156 |
convos = [] # store all panel objects in a list
|
| 157 |
|
|
|
|
| 160 |
|
| 161 |
prompt_text = prompt.value
|
| 162 |
if prompt_text:
|
| 163 |
+
|
| 164 |
+
if use_cached_repo.value:
|
| 165 |
+
result = qa_cached_tuist(query=prompt_text, chain_type=select_chain_type.value, k=select_k.value)
|
| 166 |
+
else:
|
| 167 |
+
result = qa(url_repo=url_repo.value, branch=branch_repo.value, query=prompt_text, chain_type=select_chain_type.value, k=select_k.value)
|
| 168 |
+
|
| 169 |
convos.extend([
|
| 170 |
pn.Row(
|
| 171 |
pn.panel("\U0001F60A", width=10),
|
|
|
|
| 193 |
output = pn.WidgetBox('*Output will show up here:*', qa_interactive, width=630, scroll=True)
|
| 194 |
|
| 195 |
# Layout
|
| 196 |
+
github_inputs = pn.Column(use_cached_repo, url_repo, branch_repo, width=600)
|
| 197 |
+
|
| 198 |
+
def toggle_github_inputs(event):
|
| 199 |
+
if event.new:
|
| 200 |
+
url_repo.disabled = True
|
| 201 |
+
branch_repo.disabled = True
|
| 202 |
+
else:
|
| 203 |
+
url_repo.disabled = False
|
| 204 |
+
branch_repo.disabled = False
|
| 205 |
+
|
| 206 |
+
use_cached_repo.param.watch(toggle_github_inputs, 'value')
|
| 207 |
+
|
| 208 |
widgets = pn.Row(
|
| 209 |
pn.Column(
|
| 210 |
prompt,
|
|
|
|
| 230 |
3) Type a question and click "Run".\n
|
| 231 |
|
| 232 |
"""),
|
| 233 |
+
pn.Column(github_inputs, openaikey),
|
| 234 |
+
widgets,
|
| 235 |
output,
|
|
|
|
| 236 |
|
| 237 |
).servable()
|
tuist_cached_db/chroma-collections.parquet
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2475daf5562299e9501b623d5a1d3fe3859aaffd11c4107241d0f5c3381c4318
|
| 3 |
+
size 557
|
tuist_cached_db/chroma-embeddings.parquet
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:25fe5feab4acd8a1dc5669ca66cb433084c6e37648a99eb2b2a797586c91705d
|
| 3 |
+
size 85622368
|
tuist_cached_db/index/id_to_uuid_219bcda1-1619-4ee4-8db7-38b0df6259ed.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:aa18e8ffbe67734d3ead2beb0f31479a7b94e791ed72155d964c4ccfe1a27d5e
|
| 3 |
+
size 225756
|
tuist_cached_db/index/index_219bcda1-1619-4ee4-8db7-38b0df6259ed.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8aed6c8cb81928bf00c9c63204f4fa3d94ffff35e92832341a94e3f425f3a643
|
| 3 |
+
size 43747048
|
tuist_cached_db/index/index_metadata_219bcda1-1619-4ee4-8db7-38b0df6259ed.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:939224388a2136973b7fdfac9584216a2510991112ec9cf2a2b44348a411c02b
|
| 3 |
+
size 74
|
tuist_cached_db/index/uuid_to_id_219bcda1-1619-4ee4-8db7-38b0df6259ed.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:12de70c55007b45a0eb9e8569a4b51641796734e56924df278930c9dc39332e9
|
| 3 |
+
size 263985
|