# This app allows users to ask questions about a codebase. # It uses: # - Panel for the UI # - Langchain for the chain # - Chroma for the vectorstore # For local dev env : install the following packages #!pip install langchain openai chromadb panel gitpython python-dotenv import os import panel as pn from typing import List from urllib.parse import urlparse from dotenv import load_dotenv from langchain.chains import RetrievalQA from langchain.llms import OpenAI from langchain.document_loaders import GitLoader from langchain.indexes import VectorstoreIndexCreator from langchain.text_splitter import RecursiveCharacterTextSplitter from langchain.embeddings import OpenAIEmbeddings from langchain.embeddings import HuggingFaceInstructEmbeddings from langchain.vectorstores import Chroma from langchain.chat_models import ChatOpenAI from langchain.vectorstores import Chroma from langchain.docstore.document import Document from langchain.schema import BaseRetriever #------------------------------------------------------------------------------ # Utils def get_repo_name_from_url(url): url_parsed = urlparse(url) return os.path.basename(url_parsed.path).replace(".git", "") def get_names_subdirs(path_dir): return [f.name for f in os.scandir(path_dir) if f.is_dir()] def remove_empty_subdirs(path_dir): for root, dirs, files in os.walk(path_dir, topdown=False): for name in dirs: dir_path = os.path.join(root, name) if len(os.listdir(dir_path)) == 0: os.rmdir(dir_path) def get_names_repos_already_checkout(): path_checkout = "checkout" remove_empty_subdirs(path_checkout) return get_names_subdirs(path_checkout) def get_env(key): load_dotenv() return os.getenv(key) #------------------------------------------------------------------------------ # Panel setup pn.extension( 'texteditor', template = "bootstrap", sizing_mode = 'stretch_width' ) pn.state.template.param.update( main_max_width = "690px", header_background = "#F08080", title = "Squirrel : Semantic code search and question answering" ) #------------------------------------------------------------------------------ # User inputs input_key_openai = pn.widgets.PasswordInput( value = get_env("OPENAI_API_KEY"), placeholder = "Enter your OpenAI API Key here...", width = 300 ) select_repo = pn.widgets.RadioBoxGroup( options = get_names_repos_already_checkout() + ["Custom"], inline = False ) input_url_repo = pn.widgets.TextInput( value = "", placeholder = "Enter your GitHub repo URL here...", width = 300 ) input_branch_repo = pn.widgets.TextInput( value="", placeholder="Enter the GitHub repo branch here...", width=300 ) select_embedding = pn.widgets.RadioBoxGroup( options=['openai', 'instruct'], inline=False ) input_question = pn.widgets.TextEditor( value="", placeholder="Enter your questions here...", height=200, toolbar=False ) run_button = pn.widgets.Button( name="Run", button_type="primary" ) select_k = pn.widgets.IntSlider( name="Number of relevant chunks", start=1, end=10, step=1, value=3 ) select_chain_type = pn.widgets.RadioButtonGroup( name='Chain type', options=['stuff', 'map_reduce', "refine", "map_rerank"] ) def toggle_github_inputs(event): if event.new == "Custom": input_url_repo.disabled = False input_branch_repo.disabled = False else: input_url_repo.disabled = True input_branch_repo.disabled = True select_repo.param.watch(toggle_github_inputs, 'value') #------------------------------------------------------------------------------ def init_embedding_openai(): return OpenAIEmbeddings( model = "text-embedding-ada-002", openai_api_key=input_key_openai.value ) def init_embedding_instruct(): # Default model: instructor-large return HuggingFaceInstructEmbeddings( query_instruction="Represent the query for retrival:" ) embedding_functions = { "openai": init_embedding_openai, "instruct": init_embedding_instruct } result_panel_objects = [] def load_repo(name_repo): print(f"Name repo: {name_repo}") path_checkout = os.path.join("checkout", name_repo) print(f"Path checkout: {path_checkout}") is_repo_already_checkout = os.path.exists(path_checkout) and len(os.listdir(path_checkout)) > 0 print(f"Is repo already checkout: {is_repo_already_checkout}") print(f"Repo URL: {input_url_repo.value}") print(f"Branch: {input_branch_repo.value}") file_filter = lambda file_path: file_path.endswith(".swift") or file_path.endswith(".h") or file_path.endswith(".m") if is_repo_already_checkout: print("Repo already checkout") print("Loading documents...") loader = GitLoader( repo_path=path_checkout, branch=input_branch_repo.value, file_filter=file_filter ) else: print("Repo not checkout") print("Cloning repo and loading documents...") loader = GitLoader( repo_path=path_checkout, clone_url=input_url_repo.value, branch=input_branch_repo.value, file_filter=file_filter ) documents = loader.load() return documents def get_result_qa_chain(): os.environ["OPENAI_API_KEY"] = input_key_openai.value if select_repo.value == "Custom": name_repo = get_repo_name_from_url(input_url_repo.value) else: name_repo = select_repo.value name_embedding = select_embedding.value embedding_function = embedding_functions[name_embedding]() path_dir_db = f"dbs/{name_repo}/{name_embedding}" is_repo_already_indexed = os.path.exists(path_dir_db) result_panel_objects.clear() if not is_repo_already_indexed: print("Repo not indexed. Starting indexing...") documents = load_repo(name_repo) texts = RecursiveCharacterTextSplitter(chunk_size=3000, chunk_overlap=0).split_documents(documents) db = Chroma.from_documents( documents=texts, embedding=embedding_function, persist_directory=path_dir_db ) else: print("Repo already indexed") db = Chroma( embedding_function=embedding_function, persist_directory=path_dir_db, ) db.persist() retriever = db.as_retriever( search_type = "similarity", search_kwargs = { "k": select_k.value } ) # llm=OpenAI(temperature=0, model_name="text-davinci-002") llm = ChatOpenAI( model_name = "gpt-3.5-turbo", temperature=0.7, max_tokens=1000) # create a chain to answer questions qa = RetrievalQA.from_chain_type( llm=llm, chain_type=select_chain_type.value, retriever=retriever, return_source_documents=True ) final_prompt = f"You are a coder 'Bot' that helps the 'User' reading, understanding and writing code, being proactive and as verbose as necessary. Propose code and refactor sources when needed. User: {input_question.value} Bot:" result = qa({"query": final_prompt}) return result def result_box(_): if input_question.value: result = get_result_qa_chain() result_panel_objects.extend([ pn.Row( pn.panel("\U0001F60A", width=10), input_question.value ), pn.Row( pn.panel("\U0001F916", width=10), pn.Column( result["result"], "
", pn.pane.Markdown(f"### Retrieved sources\n"), pn.Column( *[pn.pane.Markdown(f"#####{i+1}\n```\n{doc.page_content}\n```", width=600) for i, doc in enumerate(result["source_documents"])] ) ) ) ]) return pn.Column( *result_panel_objects, min_height=400, min_width=600, ) # ----------------------------------------------------------------------------- # Layout config_section = pn.Column( pn.Column( input_key_openai ), pn.Card( pn.Column( select_repo, input_url_repo, input_branch_repo ), title="Load codebase" ), pn.Card( pn.Column( select_embedding ), title="Select embedding" ), pn.Card( "Chain type:", pn.Column( select_chain_type, select_k ), title="Advanced settings", collapsed=True ) ) input_section = pn.Card( pn.Row(input_question), run_button, title="Ask a question" ) # show time elapsed output_section = pn.Card( pn.panel( pn.bind( result_box, run_button ), loading_indicator=True, ), title="Output" ) pn.Column( pn.pane.Markdown(""" ### \U0001F60A Ask Squirrel a question about your codebase 1. Enter the url of a github repo and a branch (swift / objc) 2. Enter OpenAI API key. This costs $. Set up billing at [OpenAI](https://platform.openai.com/account). 3. Type a question and click "Run". """), config_section, input_section, output_section ).servable()