Download app.py from spqw-org/code-qa: direct link, hf CLI and curl.
- Browser
- Download file 9.43 kB
-
https://huggingface.co/spaces/spqw-org/code-qa/resolve/main/app.py
- Command line
-
hf download hf://spaces/spqw-org/code-qa/app.py
-
curl -L -o app.py https://huggingface.co/spaces/spqw-org/code-qa/resolve/main/app.py
9.43 kB
| # This app allows users to ask questions about a codebase. | |
| # It uses: | |
| # - Panel for the UI | |
| # - Langchain for the chain | |
| # - Chroma for the vectorstore | |
| # For local dev env : install the following packages | |
| #!pip install langchain openai chromadb panel gitpython python-dotenv | |
| import os | |
| import panel as pn | |
| from typing import List | |
| from urllib.parse import urlparse | |
| from dotenv import load_dotenv | |
| from langchain.chains import RetrievalQA | |
| from langchain.llms import OpenAI | |
| from langchain.document_loaders import GitLoader | |
| from langchain.indexes import VectorstoreIndexCreator | |
| from langchain.text_splitter import RecursiveCharacterTextSplitter | |
| from langchain.embeddings import OpenAIEmbeddings | |
| from langchain.embeddings import HuggingFaceInstructEmbeddings | |
| from langchain.vectorstores import Chroma | |
| from langchain.chat_models import ChatOpenAI | |
| from langchain.vectorstores import Chroma | |
| from langchain.docstore.document import Document | |
| from langchain.schema import BaseRetriever | |
| #------------------------------------------------------------------------------ | |
| # Utils | |
| def get_repo_name_from_url(url): | |
| url_parsed = urlparse(url) | |
| return os.path.basename(url_parsed.path).replace(".git", "") | |
| def get_names_subdirs(path_dir): | |
| return [f.name for f in os.scandir(path_dir) if f.is_dir()] | |
| def remove_empty_subdirs(path_dir): | |
| for root, dirs, files in os.walk(path_dir, topdown=False): | |
| for name in dirs: | |
| dir_path = os.path.join(root, name) | |
| if len(os.listdir(dir_path)) == 0: | |
| os.rmdir(dir_path) | |
| def get_names_repos_already_checkout(): | |
| path_checkout = "checkout" | |
| remove_empty_subdirs(path_checkout) | |
| return get_names_subdirs(path_checkout) | |
| def get_env(key): | |
| load_dotenv() | |
| return os.getenv(key) | |
| #------------------------------------------------------------------------------ | |
| # Panel setup | |
| pn.extension( | |
| 'texteditor', | |
| template = "bootstrap", | |
| sizing_mode = 'stretch_width' | |
| ) | |
| pn.state.template.param.update( | |
| main_max_width = "690px", | |
| header_background = "#F08080", | |
| title = "Squirrel : Semantic code search and question answering" | |
| ) | |
| #------------------------------------------------------------------------------ | |
| # User inputs | |
| input_key_openai = pn.widgets.PasswordInput( | |
| value = get_env("OPENAI_API_KEY"), | |
| placeholder = "Enter your OpenAI API Key here...", | |
| width = 300 | |
| ) | |
| select_repo = pn.widgets.RadioBoxGroup( | |
| options = get_names_repos_already_checkout() + ["Custom"], | |
| inline = False | |
| ) | |
| input_url_repo = pn.widgets.TextInput( | |
| value = "", | |
| placeholder = "Enter your GitHub repo URL here...", | |
| width = 300 | |
| ) | |
| input_branch_repo = pn.widgets.TextInput( | |
| value="", | |
| placeholder="Enter the GitHub repo branch here...", | |
| width=300 | |
| ) | |
| select_embedding = pn.widgets.RadioBoxGroup( | |
| options=['openai', 'instruct'], | |
| inline=False | |
| ) | |
| input_question = pn.widgets.TextEditor( | |
| value="", | |
| placeholder="Enter your questions here...", | |
| height=200, | |
| toolbar=False | |
| ) | |
| run_button = pn.widgets.Button( | |
| name="Run", | |
| button_type="primary" | |
| ) | |
| select_k = pn.widgets.IntSlider( | |
| name="Number of relevant chunks", | |
| start=1, | |
| end=10, | |
| step=1, | |
| value=3 | |
| ) | |
| select_chain_type = pn.widgets.RadioButtonGroup( | |
| name='Chain type', | |
| options=['stuff', 'map_reduce', "refine", "map_rerank"] | |
| ) | |
| def toggle_github_inputs(event): | |
| if event.new == "Custom": | |
| input_url_repo.disabled = False | |
| input_branch_repo.disabled = False | |
| else: | |
| input_url_repo.disabled = True | |
| input_branch_repo.disabled = True | |
| select_repo.param.watch(toggle_github_inputs, 'value') | |
| #------------------------------------------------------------------------------ | |
| def init_embedding_openai(): | |
| return OpenAIEmbeddings( | |
| model = "text-embedding-ada-002", | |
| openai_api_key=input_key_openai.value | |
| ) | |
| def init_embedding_instruct(): | |
| # Default model: instructor-large | |
| return HuggingFaceInstructEmbeddings( | |
| query_instruction="Represent the query for retrival:" | |
| ) | |
| embedding_functions = { | |
| "openai": init_embedding_openai, | |
| "instruct": init_embedding_instruct | |
| } | |
| result_panel_objects = [] | |
| def load_repo(name_repo): | |
| print(f"Name repo: {name_repo}") | |
| path_checkout = os.path.join("checkout", name_repo) | |
| print(f"Path checkout: {path_checkout}") | |
| is_repo_already_checkout = os.path.exists(path_checkout) and len(os.listdir(path_checkout)) > 0 | |
| print(f"Is repo already checkout: {is_repo_already_checkout}") | |
| print(f"Repo URL: {input_url_repo.value}") | |
| print(f"Branch: {input_branch_repo.value}") | |
| file_filter = lambda file_path: file_path.endswith(".swift") or file_path.endswith(".h") or file_path.endswith(".m") | |
| if is_repo_already_checkout: | |
| print("Repo already checkout") | |
| print("Loading documents...") | |
| loader = GitLoader( | |
| repo_path=path_checkout, | |
| branch=input_branch_repo.value, | |
| file_filter=file_filter | |
| ) | |
| else: | |
| print("Repo not checkout") | |
| print("Cloning repo and loading documents...") | |
| loader = GitLoader( | |
| repo_path=path_checkout, | |
| clone_url=input_url_repo.value, | |
| branch=input_branch_repo.value, | |
| file_filter=file_filter | |
| ) | |
| documents = loader.load() | |
| return documents | |
| def get_result_qa_chain(): | |
| os.environ["OPENAI_API_KEY"] = input_key_openai.value | |
| if select_repo.value == "Custom": | |
| name_repo = get_repo_name_from_url(input_url_repo.value) | |
| else: | |
| name_repo = select_repo.value | |
| name_embedding = select_embedding.value | |
| embedding_function = embedding_functions[name_embedding]() | |
| path_dir_db = f"dbs/{name_repo}/{name_embedding}" | |
| is_repo_already_indexed = os.path.exists(path_dir_db) | |
| result_panel_objects.clear() | |
| if not is_repo_already_indexed: | |
| print("Repo not indexed. Starting indexing...") | |
| documents = load_repo(name_repo) | |
| texts = RecursiveCharacterTextSplitter(chunk_size=3000, chunk_overlap=0).split_documents(documents) | |
| db = Chroma.from_documents( | |
| documents=texts, | |
| embedding=embedding_function, | |
| persist_directory=path_dir_db | |
| ) | |
| else: | |
| print("Repo already indexed") | |
| db = Chroma( | |
| embedding_function=embedding_function, | |
| persist_directory=path_dir_db, | |
| ) | |
| db.persist() | |
| retriever = db.as_retriever( | |
| search_type = "similarity", | |
| search_kwargs = { "k": select_k.value } | |
| ) | |
| # llm=OpenAI(temperature=0, model_name="text-davinci-002") | |
| llm = ChatOpenAI( | |
| model_name = "gpt-3.5-turbo", | |
| temperature=0.7, | |
| max_tokens=1000) | |
| # create a chain to answer questions | |
| qa = RetrievalQA.from_chain_type( | |
| llm=llm, | |
| chain_type=select_chain_type.value, | |
| retriever=retriever, | |
| return_source_documents=True | |
| ) | |
| final_prompt = f"You are a coder 'Bot' that helps the 'User' reading, understanding and writing code, being proactive and as verbose as necessary. Propose code and refactor sources when needed. User: {input_question.value} Bot:" | |
| result = qa({"query": final_prompt}) | |
| return result | |
| def result_box(_): | |
| if input_question.value: | |
| result = get_result_qa_chain() | |
| result_panel_objects.extend([ | |
| pn.Row( | |
| pn.panel("\U0001F60A", width=10), | |
| input_question.value | |
| ), | |
| pn.Row( | |
| pn.panel("\U0001F916", width=10), | |
| pn.Column( | |
| result["result"], | |
| "<hr>", | |
| pn.pane.Markdown(f"### Retrieved sources\n"), | |
| pn.Column( | |
| *[pn.pane.Markdown(f"#####{i+1}\n```\n{doc.page_content}\n```", width=600) for i, doc in enumerate(result["source_documents"])] | |
| ) | |
| ) | |
| ) | |
| ]) | |
| return pn.Column( | |
| *result_panel_objects, | |
| min_height=400, | |
| min_width=600, | |
| ) | |
| # ----------------------------------------------------------------------------- | |
| # Layout | |
| config_section = pn.Column( | |
| pn.Column( | |
| input_key_openai | |
| ), | |
| pn.Card( | |
| pn.Column( | |
| select_repo, | |
| input_url_repo, | |
| input_branch_repo | |
| ), | |
| title="Load codebase" | |
| ), | |
| pn.Card( | |
| pn.Column( | |
| select_embedding | |
| ), | |
| title="Select embedding" | |
| ), | |
| pn.Card( | |
| "Chain type:", | |
| pn.Column( | |
| select_chain_type, | |
| select_k | |
| ), | |
| title="Advanced settings", | |
| collapsed=True | |
| ) | |
| ) | |
| input_section = pn.Card( | |
| pn.Row(input_question), | |
| run_button, | |
| title="Ask a question" | |
| ) | |
| # show time elapsed | |
| output_section = pn.Card( | |
| pn.panel( | |
| pn.bind( | |
| result_box, | |
| run_button | |
| ), | |
| loading_indicator=True, | |
| ), | |
| title="Output" | |
| ) | |
| pn.Column( | |
| pn.pane.Markdown(""" | |
| ### \U0001F60A Ask Squirrel a question about your codebase | |
| 1. Enter the url of a github repo and a branch (swift / objc) | |
| 2. Enter OpenAI API key. This costs $. Set up billing at [OpenAI](https://platform.openai.com/account). | |
| 3. Type a question and click "Run". | |
| """), | |
| config_section, | |
| input_section, | |
| output_section | |
| ).servable() |