Stéphane G commited on
Commit ·
da00ae1
1
Parent(s): ef46e30
refactor + add instruct embedding
Browse files- Dockerfile +1 -1
- app.py +325 -0
- checkout/BusyNavigationBar +0 -1
- checkout/tuist +0 -1
- code_qa_app.py +0 -272
- dbs/Alamofire/chroma-collections.parquet +0 -3
- dbs/{tuist_cached → Alamofire/openai}/chroma-collections.parquet +1 -1
- dbs/Alamofire/{chroma-embeddings.parquet → openai/chroma-embeddings.parquet} +2 -2
- dbs/Alamofire/{index/id_to_uuid_82e51ffe-b003-42db-a060-7ee52745f81d.pkl → openai/index/id_to_uuid_0fc9eee1-a3d5-4c9c-9ed1-059a474e69f0.pkl} +2 -2
- dbs/Alamofire/{index/index_82e51ffe-b003-42db-a060-7ee52745f81d.bin → openai/index/index_0fc9eee1-a3d5-4c9c-9ed1-059a474e69f0.bin} +1 -1
- dbs/Alamofire/{index/index_metadata_82e51ffe-b003-42db-a060-7ee52745f81d.pkl → openai/index/index_metadata_0fc9eee1-a3d5-4c9c-9ed1-059a474e69f0.pkl} +1 -1
- dbs/Alamofire/{index/uuid_to_id_82e51ffe-b003-42db-a060-7ee52745f81d.pkl → openai/index/uuid_to_id_0fc9eee1-a3d5-4c9c-9ed1-059a474e69f0.pkl} +1 -1
- dbs/tuist_cached/chroma-embeddings.parquet +0 -3
- dbs/tuist_cached/index/id_to_uuid_dbb9dc64-2123-4c5d-a69d-a08a9d0b3485.pkl +0 -3
- dbs/tuist_cached/index/index_dbb9dc64-2123-4c5d-a69d-a08a9d0b3485.bin +0 -3
- dbs/tuist_cached/index/index_metadata_dbb9dc64-2123-4c5d-a69d-a08a9d0b3485.pkl +0 -3
- dbs/tuist_cached/index/uuid_to_id_dbb9dc64-2123-4c5d-a69d-a08a9d0b3485.pkl +0 -3
- requirements.txt +2 -1
- scripts/dev_up.sh +1 -1
Dockerfile
CHANGED
|
@@ -8,7 +8,7 @@ RUN python3 -m pip install --no-cache-dir --upgrade -r /code/requirements.txt
|
|
| 8 |
|
| 9 |
COPY . .
|
| 10 |
|
| 11 |
-
CMD ["panel", "serve", "/code/
|
| 12 |
|
| 13 |
RUN mkdir /.cache
|
| 14 |
RUN chmod 777 /.cache
|
|
|
|
| 8 |
|
| 9 |
COPY . .
|
| 10 |
|
| 11 |
+
CMD ["panel", "serve", "/code/app.py", "--address", "0.0.0.0", "--port", "7860", "--allow-websocket-origin", "spqw-org-code-qa.hf.space", "--allow-websocket-origin", "0.0.0.0:7860"]
|
| 12 |
|
| 13 |
RUN mkdir /.cache
|
| 14 |
RUN chmod 777 /.cache
|
app.py
ADDED
|
@@ -0,0 +1,325 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# This app allows users to ask questions about a codebase.
|
| 2 |
+
# It uses:
|
| 3 |
+
# - Panel for the UI
|
| 4 |
+
# - Langchain for the chain
|
| 5 |
+
# - Chroma for the vectorstore
|
| 6 |
+
|
| 7 |
+
# For local dev env : install the following packages
|
| 8 |
+
#!pip install langchain openai chromadb panel gitpython
|
| 9 |
+
|
| 10 |
+
import os
|
| 11 |
+
import panel as pn
|
| 12 |
+
from typing import List
|
| 13 |
+
from urllib.parse import urlparse
|
| 14 |
+
|
| 15 |
+
from langchain.chains import RetrievalQA
|
| 16 |
+
from langchain.llms import OpenAI
|
| 17 |
+
from langchain.document_loaders import GitLoader
|
| 18 |
+
from langchain.indexes import VectorstoreIndexCreator
|
| 19 |
+
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
| 20 |
+
from langchain.embeddings import OpenAIEmbeddings
|
| 21 |
+
from langchain.embeddings import HuggingFaceInstructEmbeddings
|
| 22 |
+
from langchain.vectorstores import Chroma
|
| 23 |
+
from langchain.chat_models import ChatOpenAI
|
| 24 |
+
from langchain.vectorstores import Chroma
|
| 25 |
+
from langchain.docstore.document import Document
|
| 26 |
+
from langchain.schema import BaseRetriever
|
| 27 |
+
|
| 28 |
+
#------------------------------------------------------------------------------
|
| 29 |
+
# Utils
|
| 30 |
+
|
| 31 |
+
def get_repo_name_from_url(url):
|
| 32 |
+
url_parsed = urlparse(url)
|
| 33 |
+
return os.path.basename(url_parsed.path).replace(".git", "")
|
| 34 |
+
|
| 35 |
+
def get_names_subdirs(path_dir):
|
| 36 |
+
return [f.name for f in os.scandir(path_dir) if f.is_dir()]
|
| 37 |
+
|
| 38 |
+
def remove_empty_subdirs(path_dir):
|
| 39 |
+
for root, dirs, files in os.walk(path_dir, topdown=False):
|
| 40 |
+
for name in dirs:
|
| 41 |
+
dir_path = os.path.join(root, name)
|
| 42 |
+
if len(os.listdir(dir_path)) == 0:
|
| 43 |
+
os.rmdir(dir_path)
|
| 44 |
+
|
| 45 |
+
def get_names_repos_already_checkout():
|
| 46 |
+
path_checkout = "checkout"
|
| 47 |
+
remove_empty_subdirs(path_checkout)
|
| 48 |
+
return get_names_subdirs(path_checkout)
|
| 49 |
+
|
| 50 |
+
#------------------------------------------------------------------------------
|
| 51 |
+
# Panel setup
|
| 52 |
+
pn.extension(
|
| 53 |
+
'texteditor',
|
| 54 |
+
template = "bootstrap",
|
| 55 |
+
sizing_mode = 'stretch_width'
|
| 56 |
+
)
|
| 57 |
+
|
| 58 |
+
pn.state.template.param.update(
|
| 59 |
+
main_max_width = "690px",
|
| 60 |
+
header_background = "#F08080",
|
| 61 |
+
title = "Squirrel : Semantic code search and question answering"
|
| 62 |
+
)
|
| 63 |
+
|
| 64 |
+
#------------------------------------------------------------------------------
|
| 65 |
+
# User inputs
|
| 66 |
+
|
| 67 |
+
input_key_openai = pn.widgets.PasswordInput(
|
| 68 |
+
value = "",
|
| 69 |
+
placeholder = "Enter your OpenAI API Key here...",
|
| 70 |
+
width = 300
|
| 71 |
+
)
|
| 72 |
+
|
| 73 |
+
select_repo = pn.widgets.RadioBoxGroup(
|
| 74 |
+
options = get_names_repos_already_checkout() + ["Custom"],
|
| 75 |
+
inline = False
|
| 76 |
+
)
|
| 77 |
+
|
| 78 |
+
input_url_repo = pn.widgets.TextInput(
|
| 79 |
+
value = "",
|
| 80 |
+
placeholder = "Enter your GitHub repo URL here...",
|
| 81 |
+
width = 300
|
| 82 |
+
)
|
| 83 |
+
|
| 84 |
+
input_branch_repo = pn.widgets.TextInput(
|
| 85 |
+
value="",
|
| 86 |
+
placeholder="Enter the GitHub repo branch here...",
|
| 87 |
+
width=300
|
| 88 |
+
)
|
| 89 |
+
|
| 90 |
+
select_embedding = pn.widgets.RadioBoxGroup(
|
| 91 |
+
options=['openai', 'instruct'],
|
| 92 |
+
inline=False
|
| 93 |
+
)
|
| 94 |
+
|
| 95 |
+
input_question = pn.widgets.TextEditor(
|
| 96 |
+
value="",
|
| 97 |
+
placeholder="Enter your questions here...",
|
| 98 |
+
height=200,
|
| 99 |
+
toolbar=False
|
| 100 |
+
)
|
| 101 |
+
|
| 102 |
+
run_button = pn.widgets.Button(
|
| 103 |
+
name="Run",
|
| 104 |
+
button_type="primary"
|
| 105 |
+
)
|
| 106 |
+
|
| 107 |
+
select_k = pn.widgets.IntSlider(
|
| 108 |
+
name="Number of relevant chunks",
|
| 109 |
+
start=1,
|
| 110 |
+
end=10,
|
| 111 |
+
step=1,
|
| 112 |
+
value=3
|
| 113 |
+
)
|
| 114 |
+
|
| 115 |
+
select_chain_type = pn.widgets.RadioButtonGroup(
|
| 116 |
+
name='Chain type',
|
| 117 |
+
options=['stuff', 'map_reduce', "refine", "map_rerank"]
|
| 118 |
+
)
|
| 119 |
+
|
| 120 |
+
def toggle_github_inputs(event):
|
| 121 |
+
if event.new == "Custom":
|
| 122 |
+
input_url_repo.visible = True
|
| 123 |
+
input_branch_repo.visible = True
|
| 124 |
+
else:
|
| 125 |
+
input_url_repo.visible = False
|
| 126 |
+
input_branch_repo.visible = False
|
| 127 |
+
|
| 128 |
+
select_repo.param.watch(toggle_github_inputs, 'value')
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
#------------------------------------------------------------------------------
|
| 132 |
+
def init_embedding_openai():
|
| 133 |
+
return OpenAIEmbeddings(
|
| 134 |
+
model = "text-embedding-ada-002",
|
| 135 |
+
openai_api_key=input_key_openai.value
|
| 136 |
+
)
|
| 137 |
+
|
| 138 |
+
def init_embedding_instruct():
|
| 139 |
+
# Default model: instructor-large
|
| 140 |
+
return HuggingFaceInstructEmbeddings(
|
| 141 |
+
query_instruction="Represent the query for retrival:"
|
| 142 |
+
)
|
| 143 |
+
|
| 144 |
+
embedding_functions = {
|
| 145 |
+
"openai": init_embedding_openai,
|
| 146 |
+
"instruct": init_embedding_instruct
|
| 147 |
+
}
|
| 148 |
+
|
| 149 |
+
result_panel_objects = []
|
| 150 |
+
|
| 151 |
+
def load_repo(name_repo):
|
| 152 |
+
print(f"Name repo: {name_repo}")
|
| 153 |
+
path_checkout = os.path.join("checkout", name_repo)
|
| 154 |
+
print(f"Path checkout: {path_checkout}")
|
| 155 |
+
is_repo_already_checkout = os.path.exists(path_checkout) and len(os.listdir(path_checkout)) > 0
|
| 156 |
+
print(f"Is repo already checkout: {is_repo_already_checkout}")
|
| 157 |
+
print(f"Repo URL: {input_url_repo.value}")
|
| 158 |
+
print(f"Branch: {input_branch_repo.value}")
|
| 159 |
+
|
| 160 |
+
file_filter = lambda file_path: file_path.endswith(".swift") or file_path.endswith(".h") or file_path.endswith(".m")
|
| 161 |
+
|
| 162 |
+
if is_repo_already_checkout:
|
| 163 |
+
print("Repo already checkout")
|
| 164 |
+
print("Loading documents...")
|
| 165 |
+
loader = GitLoader(
|
| 166 |
+
repo_path=path_checkout,
|
| 167 |
+
branch=input_branch_repo.value,
|
| 168 |
+
file_filter=file_filter
|
| 169 |
+
)
|
| 170 |
+
else:
|
| 171 |
+
print("Repo not checkout")
|
| 172 |
+
print("Cloning repo and loading documents...")
|
| 173 |
+
loader = GitLoader(
|
| 174 |
+
repo_path=path_checkout,
|
| 175 |
+
clone_url=input_url_repo.value,
|
| 176 |
+
branch=input_branch_repo.value,
|
| 177 |
+
file_filter=file_filter
|
| 178 |
+
)
|
| 179 |
+
|
| 180 |
+
documents = loader.load()
|
| 181 |
+
return documents
|
| 182 |
+
|
| 183 |
+
def get_result_qa_chain():
|
| 184 |
+
os.environ["OPENAI_API_KEY"] = input_key_openai.value
|
| 185 |
+
if select_repo.value == "Custom":
|
| 186 |
+
name_repo = get_repo_name_from_url(input_url_repo.value)
|
| 187 |
+
else:
|
| 188 |
+
name_repo = select_repo.value
|
| 189 |
+
name_embedding = select_embedding.value
|
| 190 |
+
embedding_function = embedding_functions[name_embedding]()
|
| 191 |
+
path_dir_db = f"dbs/{name_repo}/{name_embedding}"
|
| 192 |
+
is_repo_already_indexed = os.path.exists(path_dir_db)
|
| 193 |
+
|
| 194 |
+
result_panel_objects.clear()
|
| 195 |
+
|
| 196 |
+
if not is_repo_already_indexed:
|
| 197 |
+
print("Repo not indexed. Starting indexing...")
|
| 198 |
+
documents = load_repo(name_repo)
|
| 199 |
+
texts = RecursiveCharacterTextSplitter(chunk_size=3000, chunk_overlap=0).split_documents(documents)
|
| 200 |
+
db = Chroma.from_documents(
|
| 201 |
+
documents=texts,
|
| 202 |
+
embedding=embedding_function,
|
| 203 |
+
persist_directory=path_dir_db
|
| 204 |
+
)
|
| 205 |
+
else:
|
| 206 |
+
print("Repo already indexed")
|
| 207 |
+
db = Chroma(
|
| 208 |
+
embedding_function=embedding_function,
|
| 209 |
+
persist_directory=path_dir_db,
|
| 210 |
+
)
|
| 211 |
+
db.persist()
|
| 212 |
+
|
| 213 |
+
retriever = db.as_retriever(
|
| 214 |
+
search_type = "similarity",
|
| 215 |
+
search_kwargs = { "k": select_k.value }
|
| 216 |
+
)
|
| 217 |
+
|
| 218 |
+
# llm=OpenAI(temperature=0, model_name="text-davinci-002")
|
| 219 |
+
llm = ChatOpenAI(
|
| 220 |
+
model_name = "gpt-3.5-turbo",
|
| 221 |
+
temperature=0.7,
|
| 222 |
+
max_tokens=1000)
|
| 223 |
+
|
| 224 |
+
# create a chain to answer questions
|
| 225 |
+
qa = RetrievalQA.from_chain_type(
|
| 226 |
+
llm=llm,
|
| 227 |
+
chain_type=select_chain_type.value,
|
| 228 |
+
retriever=retriever,
|
| 229 |
+
return_source_documents=True
|
| 230 |
+
)
|
| 231 |
+
|
| 232 |
+
result = qa({"query": input_question.value})
|
| 233 |
+
return result
|
| 234 |
+
|
| 235 |
+
def result_box(_):
|
| 236 |
+
if input_question.value:
|
| 237 |
+
result = get_result_qa_chain()
|
| 238 |
+
|
| 239 |
+
result_panel_objects.extend([
|
| 240 |
+
pn.Row(
|
| 241 |
+
pn.panel("\U0001F60A", width=10),
|
| 242 |
+
input_question.value
|
| 243 |
+
),
|
| 244 |
+
pn.Row(
|
| 245 |
+
pn.panel("\U0001F916", width=10),
|
| 246 |
+
pn.Column(
|
| 247 |
+
result["result"],
|
| 248 |
+
"<hr>",
|
| 249 |
+
pn.pane.Markdown(f"### Retrieved sources\n"),
|
| 250 |
+
pn.Column(
|
| 251 |
+
*[pn.pane.Markdown(f"#####{i+1}\n```\n{doc.page_content}\n```", width=600) for i, doc in enumerate(result["source_documents"])]
|
| 252 |
+
)
|
| 253 |
+
)
|
| 254 |
+
)
|
| 255 |
+
])
|
| 256 |
+
|
| 257 |
+
|
| 258 |
+
return pn.Column(
|
| 259 |
+
*result_panel_objects,
|
| 260 |
+
min_height=400,
|
| 261 |
+
min_width=600,
|
| 262 |
+
)
|
| 263 |
+
|
| 264 |
+
# -----------------------------------------------------------------------------
|
| 265 |
+
# Layout
|
| 266 |
+
config_section = pn.Column(
|
| 267 |
+
pn.Column(
|
| 268 |
+
input_key_openai
|
| 269 |
+
),
|
| 270 |
+
pn.Card(
|
| 271 |
+
pn.Column(
|
| 272 |
+
select_repo,
|
| 273 |
+
input_url_repo,
|
| 274 |
+
input_branch_repo
|
| 275 |
+
),
|
| 276 |
+
title="Load codebase"
|
| 277 |
+
),
|
| 278 |
+
pn.Card(
|
| 279 |
+
pn.Column(
|
| 280 |
+
select_embedding
|
| 281 |
+
),
|
| 282 |
+
title="Select embedding"
|
| 283 |
+
),
|
| 284 |
+
pn.Card(
|
| 285 |
+
"Chain type:",
|
| 286 |
+
pn.Column(
|
| 287 |
+
select_chain_type,
|
| 288 |
+
select_k
|
| 289 |
+
),
|
| 290 |
+
title="Advanced settings",
|
| 291 |
+
collapsed=True
|
| 292 |
+
)
|
| 293 |
+
)
|
| 294 |
+
|
| 295 |
+
input_section = pn.Card(
|
| 296 |
+
pn.Row(input_question),
|
| 297 |
+
run_button,
|
| 298 |
+
title="Ask a question"
|
| 299 |
+
)
|
| 300 |
+
|
| 301 |
+
# show time elapsed
|
| 302 |
+
output_section = pn.Card(
|
| 303 |
+
pn.panel(
|
| 304 |
+
pn.bind(
|
| 305 |
+
result_box,
|
| 306 |
+
run_button
|
| 307 |
+
),
|
| 308 |
+
loading_indicator=True,
|
| 309 |
+
),
|
| 310 |
+
title="Output"
|
| 311 |
+
)
|
| 312 |
+
|
| 313 |
+
pn.Column(
|
| 314 |
+
pn.pane.Markdown("""
|
| 315 |
+
### \U0001F60A Ask Squirrel a question about your codebase
|
| 316 |
+
|
| 317 |
+
1. Enter the url of a github repo and a branch (swift / objc)
|
| 318 |
+
2. Enter OpenAI API key. This costs $. Set up billing at [OpenAI](https://platform.openai.com/account).
|
| 319 |
+
3. Type a question and click "Run".
|
| 320 |
+
|
| 321 |
+
"""),
|
| 322 |
+
config_section,
|
| 323 |
+
input_section,
|
| 324 |
+
output_section
|
| 325 |
+
).servable()
|
checkout/BusyNavigationBar
DELETED
|
@@ -1 +0,0 @@
|
|
| 1 |
-
Subproject commit 48059ab3bbf4159d71e8ce69c62c51ae284303c7
|
|
|
|
|
|
checkout/tuist
DELETED
|
@@ -1 +0,0 @@
|
|
| 1 |
-
Subproject commit 099e3b79347a70822dcb3d10138f50787c4fd0e9
|
|
|
|
|
|
code_qa_app.py
DELETED
|
@@ -1,272 +0,0 @@
|
|
| 1 |
-
#!pip install langchain openai chromadb tiktoken panel gitpython
|
| 2 |
-
|
| 3 |
-
import os
|
| 4 |
-
import panel as pn
|
| 5 |
-
from typing import List
|
| 6 |
-
from urllib.parse import urlparse
|
| 7 |
-
|
| 8 |
-
from langchain.chains import RetrievalQA
|
| 9 |
-
from langchain.llms import OpenAI
|
| 10 |
-
from langchain.document_loaders import GitLoader
|
| 11 |
-
from langchain.indexes import VectorstoreIndexCreator
|
| 12 |
-
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
| 13 |
-
from langchain.embeddings import OpenAIEmbeddings
|
| 14 |
-
from langchain.vectorstores import Chroma
|
| 15 |
-
from langchain.chat_models import ChatOpenAI
|
| 16 |
-
from langchain.vectorstores import Chroma
|
| 17 |
-
from langchain.docstore.document import Document
|
| 18 |
-
from langchain.schema import BaseRetriever
|
| 19 |
-
|
| 20 |
-
# UI configuration
|
| 21 |
-
pn.extension(
|
| 22 |
-
'texteditor',
|
| 23 |
-
template="bootstrap",
|
| 24 |
-
sizing_mode='stretch_width'
|
| 25 |
-
)
|
| 26 |
-
|
| 27 |
-
pn.state.template.param.update(
|
| 28 |
-
main_max_width="690px",
|
| 29 |
-
header_background="#F08080",
|
| 30 |
-
)
|
| 31 |
-
|
| 32 |
-
# User inputs
|
| 33 |
-
use_cached_repo = pn.widgets.Checkbox(name="Use cached Alamofire repo instead")
|
| 34 |
-
|
| 35 |
-
url_repo = pn.widgets.TextInput(
|
| 36 |
-
value="",
|
| 37 |
-
placeholder="Enter your GitHub repo URL here...",
|
| 38 |
-
width=300
|
| 39 |
-
)
|
| 40 |
-
|
| 41 |
-
branch_repo = pn.widgets.TextInput(
|
| 42 |
-
value="",
|
| 43 |
-
placeholder="Enter the GitHub repo branch here...",
|
| 44 |
-
width=300
|
| 45 |
-
)
|
| 46 |
-
|
| 47 |
-
openaikey = pn.widgets.PasswordInput(
|
| 48 |
-
value="",
|
| 49 |
-
placeholder="Enter your OpenAI API Key here...",
|
| 50 |
-
width=300
|
| 51 |
-
)
|
| 52 |
-
|
| 53 |
-
context_texteditor = pn.widgets.TextEditor(
|
| 54 |
-
value="",
|
| 55 |
-
placeholder="(Optional) Enter your index query here...",
|
| 56 |
-
height=200,
|
| 57 |
-
toolbar=False
|
| 58 |
-
)
|
| 59 |
-
|
| 60 |
-
prompt_texteditor = pn.widgets.TextEditor(
|
| 61 |
-
value="",
|
| 62 |
-
placeholder="Enter your questions here...",
|
| 63 |
-
height=200,
|
| 64 |
-
toolbar=False
|
| 65 |
-
)
|
| 66 |
-
|
| 67 |
-
run_button = pn.widgets.Button(name="Run!")
|
| 68 |
-
|
| 69 |
-
select_k = pn.widgets.IntSlider(
|
| 70 |
-
name="Number of relevant chunks",
|
| 71 |
-
start=1,
|
| 72 |
-
end=5,
|
| 73 |
-
step=1,
|
| 74 |
-
value=2
|
| 75 |
-
)
|
| 76 |
-
|
| 77 |
-
select_chain_type = pn.widgets.RadioButtonGroup(
|
| 78 |
-
name='Chain type',
|
| 79 |
-
options=['stuff', 'map_reduce', "refine", "map_rerank"]
|
| 80 |
-
)
|
| 81 |
-
|
| 82 |
-
class FixedRetriever(BaseRetriever):
|
| 83 |
-
def __init__(self, documents: List[Document]):
|
| 84 |
-
self.documents = documents
|
| 85 |
-
|
| 86 |
-
def get_relevant_documents(self, query: str):
|
| 87 |
-
# Ignoring query
|
| 88 |
-
return self.documents
|
| 89 |
-
|
| 90 |
-
async def aget_relevant_documents(self, query: str) -> List[Document]:
|
| 91 |
-
# Ignoring query
|
| 92 |
-
return self.documents
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
# Question answering function
|
| 96 |
-
def qa_fetching_repo(url_repo, branch, context_query, prompt_query, chain_type, k):
|
| 97 |
-
print("* QA fetching repo")
|
| 98 |
-
# Extract the repo name from the URL
|
| 99 |
-
url_parsed = urlparse(url_repo)
|
| 100 |
-
name_repo = os.path.basename(url_parsed.path).replace(".git", "")
|
| 101 |
-
|
| 102 |
-
# Check if the repo checkout folder exists and create it if not
|
| 103 |
-
path_repo = os.path.join("checkout", name_repo)
|
| 104 |
-
is_repo_already_checkout = False
|
| 105 |
-
if os.path.exists(path_repo):
|
| 106 |
-
is_repo_already_checkout = True
|
| 107 |
-
else:
|
| 108 |
-
os.makedirs(path_repo)
|
| 109 |
-
|
| 110 |
-
if is_repo_already_checkout:
|
| 111 |
-
loader = GitLoader(
|
| 112 |
-
repo_path=path_repo,
|
| 113 |
-
branch=branch,
|
| 114 |
-
file_filter=lambda file_path: file_path.endswith(".swift") or file_path.endswith(".h") or file_path.endswith(".m")
|
| 115 |
-
)
|
| 116 |
-
else:
|
| 117 |
-
loader = GitLoader(
|
| 118 |
-
clone_url=url_repo,
|
| 119 |
-
repo_path=path_repo,
|
| 120 |
-
branch=branch,
|
| 121 |
-
file_filter=lambda file_path: file_path.endswith(".swift") or file_path.endswith(".h") or file_path.endswith(".m")
|
| 122 |
-
)
|
| 123 |
-
|
| 124 |
-
documents = loader.load()
|
| 125 |
-
|
| 126 |
-
# split documents into chunks
|
| 127 |
-
text_splitter = RecursiveCharacterTextSplitter(chunk_size=3000, chunk_overlap=0)
|
| 128 |
-
texts = text_splitter.split_documents(documents)
|
| 129 |
-
|
| 130 |
-
# select which embeddings to use
|
| 131 |
-
embeddings = OpenAIEmbeddings()
|
| 132 |
-
|
| 133 |
-
persist_directory = os.path.join("dbs", name_repo)
|
| 134 |
-
|
| 135 |
-
# create the vectorstore to use as the index
|
| 136 |
-
db = Chroma.from_documents(texts, embeddings, persist_directory=persist_directory)
|
| 137 |
-
db.persist()
|
| 138 |
-
|
| 139 |
-
return qa(db, context_query, prompt_query, chain_type, k)
|
| 140 |
-
|
| 141 |
-
# Question answering function
|
| 142 |
-
def qa_cached_tuist(context_query, prompt_query, chain_type, k):
|
| 143 |
-
|
| 144 |
-
print("* QA cached Tuist")
|
| 145 |
-
db = Chroma(persist_directory="dbs/Alamofire/", embedding_function=OpenAIEmbeddings())
|
| 146 |
-
# db = Chroma(persist_directory="data/tuist_cached/", embedding_function=OpenAIEmbeddings())
|
| 147 |
-
|
| 148 |
-
return qa(db, context_query, prompt_query, chain_type, k)
|
| 149 |
-
|
| 150 |
-
def qa(db, context_query, prompt_query, chain_type, k):
|
| 151 |
-
|
| 152 |
-
if context_query.strip():
|
| 153 |
-
print("* Using FixedRetriver")
|
| 154 |
-
docs = db.similarity_search(context_query, k=k)
|
| 155 |
-
retriever = FixedRetriever(docs)
|
| 156 |
-
# You can test how an empty retriever makes our LLM ignorant about the context
|
| 157 |
-
# retriever = FixedRetriever([])
|
| 158 |
-
|
| 159 |
-
else:
|
| 160 |
-
print("* Using db as retriever")
|
| 161 |
-
# expose this index in a retriever interface
|
| 162 |
-
retriever = db.as_retriever(search_type="similarity", search_kwargs={"k": k})
|
| 163 |
-
|
| 164 |
-
# llm=OpenAI(temperature=0, model_name="text-davinci-002")
|
| 165 |
-
llm = ChatOpenAI(
|
| 166 |
-
model_name = "gpt-3.5-turbo",
|
| 167 |
-
temperature=0.7,
|
| 168 |
-
max_tokens=1000)
|
| 169 |
-
|
| 170 |
-
# create a chain to answer questions
|
| 171 |
-
qa = RetrievalQA.from_chain_type(
|
| 172 |
-
llm=llm,
|
| 173 |
-
chain_type=chain_type,
|
| 174 |
-
retriever=retriever,
|
| 175 |
-
return_source_documents=True
|
| 176 |
-
)
|
| 177 |
-
|
| 178 |
-
result = qa({"query": prompt_query})
|
| 179 |
-
print(result['result'])
|
| 180 |
-
return result
|
| 181 |
-
|
| 182 |
-
convos = [] # store all panel objects in a list
|
| 183 |
-
|
| 184 |
-
def qa_result(_):
|
| 185 |
-
os.environ["OPENAI_API_KEY"] = openaikey.value
|
| 186 |
-
|
| 187 |
-
prompt_text = prompt_texteditor.value
|
| 188 |
-
context_text = context_texteditor.value
|
| 189 |
-
if prompt_text:
|
| 190 |
-
|
| 191 |
-
convos.clear()
|
| 192 |
-
|
| 193 |
-
if use_cached_repo.value:
|
| 194 |
-
result = qa_cached_tuist(context_query=context_text, prompt_query=prompt_text, chain_type=select_chain_type.value, k=select_k.value)
|
| 195 |
-
else:
|
| 196 |
-
result = qa_fetching_repo(url_repo=url_repo.value, branch=branch_repo.value, context_query=context_text, prompt_query=prompt_text, chain_type=select_chain_type.value, k=select_k.value)
|
| 197 |
-
|
| 198 |
-
convos.extend([
|
| 199 |
-
pn.Row(
|
| 200 |
-
pn.panel("\U0001F60A", width=10),
|
| 201 |
-
prompt_text,
|
| 202 |
-
width=600
|
| 203 |
-
),
|
| 204 |
-
pn.Row(
|
| 205 |
-
pn.panel("\U0001F916", width=10),
|
| 206 |
-
pn.Column(
|
| 207 |
-
result["result"],
|
| 208 |
-
"From the following retrieved sources:",
|
| 209 |
-
pn.pane.Markdown(
|
| 210 |
-
'\n<hr>\n'.join(doc.page_content for doc in result["source_documents"])
|
| 211 |
-
)
|
| 212 |
-
)
|
| 213 |
-
)
|
| 214 |
-
])
|
| 215 |
-
#return convos
|
| 216 |
-
return pn.Column(*convos, margin=15, width=575, min_height=400)
|
| 217 |
-
|
| 218 |
-
qa_interactive = pn.panel(
|
| 219 |
-
pn.bind(qa_result, run_button),
|
| 220 |
-
loading_indicator=True,
|
| 221 |
-
)
|
| 222 |
-
|
| 223 |
-
# Output
|
| 224 |
-
output = pn.WidgetBox('*Output will show up here:*', qa_interactive, width=1000, scroll=True)
|
| 225 |
-
|
| 226 |
-
# Layout
|
| 227 |
-
github_inputs = pn.Column(use_cached_repo, url_repo, branch_repo, width=600)
|
| 228 |
-
|
| 229 |
-
def toggle_github_inputs(event):
|
| 230 |
-
if event.new:
|
| 231 |
-
url_repo.disabled = True
|
| 232 |
-
branch_repo.disabled = True
|
| 233 |
-
else:
|
| 234 |
-
url_repo.disabled = False
|
| 235 |
-
branch_repo.disabled = False
|
| 236 |
-
|
| 237 |
-
use_cached_repo.param.watch(toggle_github_inputs, 'value')
|
| 238 |
-
|
| 239 |
-
setup = pn.Column(
|
| 240 |
-
pn.Column(github_inputs, openaikey),
|
| 241 |
-
pn.Card(
|
| 242 |
-
"Chain type:",
|
| 243 |
-
pn.Column(
|
| 244 |
-
select_chain_type,
|
| 245 |
-
select_k
|
| 246 |
-
),
|
| 247 |
-
title="Advanced settings",
|
| 248 |
-
margin=10,
|
| 249 |
-
collapsed=True
|
| 250 |
-
)
|
| 251 |
-
)
|
| 252 |
-
|
| 253 |
-
queries = pn.Column(
|
| 254 |
-
pn.Row(prompt_texteditor),
|
| 255 |
-
# pn.Row(context_texteditor, prompt_texteditor),
|
| 256 |
-
run_button,
|
| 257 |
-
margin=5)
|
| 258 |
-
|
| 259 |
-
pn.Column(
|
| 260 |
-
pn.pane.Markdown("""
|
| 261 |
-
### \U0001F60A Ask Squirrel a question about your codebase
|
| 262 |
-
|
| 263 |
-
1. Enter the url of a github repo and a branch (swift / objc)
|
| 264 |
-
2. Enter OpenAI API key. This costs $. Set up billing at [OpenAI](https://platform.openai.com/account).
|
| 265 |
-
3. Type a question and click "Run".
|
| 266 |
-
|
| 267 |
-
"""),
|
| 268 |
-
setup,
|
| 269 |
-
queries,
|
| 270 |
-
output,
|
| 271 |
-
|
| 272 |
-
).servable()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
dbs/Alamofire/chroma-collections.parquet
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:ecbe4963d18e41866d6702e52613e682bb31f25ebb4033b8fc9dcc4b2a6f63cb
|
| 3 |
-
size 557
|
|
|
|
|
|
|
|
|
|
|
|
dbs/{tuist_cached → Alamofire/openai}/chroma-collections.parquet
RENAMED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 557
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d0589b2a7968096a032b7df9b36d957d867fa8fd54131d632a99fd3ac295608f
|
| 3 |
size 557
|
dbs/Alamofire/{chroma-embeddings.parquet → openai/chroma-embeddings.parquet}
RENAMED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0c3c8b34bd40f26227017233845625c47864e4b3740e1750bddb74849e5749a5
|
| 3 |
+
size 7722000
|
dbs/Alamofire/{index/id_to_uuid_82e51ffe-b003-42db-a060-7ee52745f81d.pkl → openai/index/id_to_uuid_0fc9eee1-a3d5-4c9c-9ed1-059a474e69f0.pkl}
RENAMED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:fe5335409f03267d3f9edf0f1c94ff49f697dd9ca04832c056cd991ef4a77a3c
|
| 3 |
+
size 19442
|
dbs/Alamofire/{index/index_82e51ffe-b003-42db-a060-7ee52745f81d.bin → openai/index/index_0fc9eee1-a3d5-4c9c-9ed1-059a474e69f0.bin}
RENAMED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 3807056
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0e196b2b00392a1f9fa59943e7cc8cff332abb5690026225e16575895010c929
|
| 3 |
size 3807056
|
dbs/Alamofire/{index/index_metadata_82e51ffe-b003-42db-a060-7ee52745f81d.pkl → openai/index/index_metadata_0fc9eee1-a3d5-4c9c-9ed1-059a474e69f0.pkl}
RENAMED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 74
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:555e4f40a0c1b6b3fdc644d2bf2084f17510b62165d6d1a4c3b1a2dce60effc2
|
| 3 |
size 74
|
dbs/Alamofire/{index/uuid_to_id_82e51ffe-b003-42db-a060-7ee52745f81d.pkl → openai/index/uuid_to_id_0fc9eee1-a3d5-4c9c-9ed1-059a474e69f0.pkl}
RENAMED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 22751
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c790ac7824f6efade0078a5b7b0902ced62be81de74fe32f21b458a212773400
|
| 3 |
size 22751
|
dbs/tuist_cached/chroma-embeddings.parquet
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:db53078400ba87eba743c80845b7fae4b53b583a2619b950569d0260c8ac3cb1
|
| 3 |
-
size 88813345
|
|
|
|
|
|
|
|
|
|
|
|
dbs/tuist_cached/index/id_to_uuid_dbb9dc64-2123-4c5d-a69d-a08a9d0b3485.pkl
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:cfb2122fac683f50360e99bda40bd42f7da8783c3dc1e7191b847b1514a12131
|
| 3 |
-
size 234446
|
|
|
|
|
|
|
|
|
|
|
|
dbs/tuist_cached/index/index_dbb9dc64-2123-4c5d-a69d-a08a9d0b3485.bin
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:79e9ad6271540c9e91126aaffb2941ca5e29749eb79df26f9fc11fce7e5d1602
|
| 3 |
-
size 45434068
|
|
|
|
|
|
|
|
|
|
|
|
dbs/tuist_cached/index/index_metadata_dbb9dc64-2123-4c5d-a69d-a08a9d0b3485.pkl
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:22ff307c21335897bc15d472a59a7f6c77f977d66eda2a6271c9f0e0f5857c5d
|
| 3 |
-
size 74
|
|
|
|
|
|
|
|
|
|
|
|
dbs/tuist_cached/index/uuid_to_id_dbb9dc64-2123-4c5d-a69d-a08a9d0b3485.pkl
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:e1e04090d384ae09a2c1f6d954363767aafd2a6d36d07dacf9fd93b274836642
|
| 3 |
-
size 274171
|
|
|
|
|
|
|
|
|
|
|
|
requirements.txt
CHANGED
|
@@ -5,4 +5,5 @@ pypdf
|
|
| 5 |
tiktoken
|
| 6 |
panel
|
| 7 |
notebook
|
| 8 |
-
gitpython
|
|
|
|
|
|
| 5 |
tiktoken
|
| 6 |
panel
|
| 7 |
notebook
|
| 8 |
+
gitpython
|
| 9 |
+
InstructorEmbedding
|
scripts/dev_up.sh
CHANGED
|
@@ -5,4 +5,4 @@ set -o pipefail
|
|
| 5 |
set -o nounset
|
| 6 |
set -o xtrace
|
| 7 |
|
| 8 |
-
panel serve
|
|
|
|
| 5 |
set -o nounset
|
| 6 |
set -o xtrace
|
| 7 |
|
| 8 |
+
panel serve app.py --address 0.0.0.0 --port 7860 --allow-websocket-origin 0.0.0.0:7860
|