Stéphane G commited on
Commit ·
040ae1b
1
Parent(s): 04a5875
add logic to checkout repos in subfolders, and avoid cloning if already checked out
Browse files- code_qa_app.py +32 -11
code_qa_app.py
CHANGED
|
@@ -11,6 +11,7 @@ from langchain.vectorstores import Chroma
|
|
| 11 |
from langchain.chat_models import ChatOpenAI
|
| 12 |
from typing import List
|
| 13 |
from langchain.vectorstores import Chroma
|
|
|
|
| 14 |
from langchain.docstore.document import Document
|
| 15 |
from langchain.schema import BaseRetriever
|
| 16 |
import panel as pn
|
|
@@ -48,7 +49,6 @@ openaikey = pn.widgets.PasswordInput(
|
|
| 48 |
width=300
|
| 49 |
)
|
| 50 |
|
| 51 |
-
context_texteditor = pn.widgets.TextEditor(
|
| 52 |
value="",
|
| 53 |
placeholder="(Optional) Enter your index query here...",
|
| 54 |
height=200,
|
|
@@ -92,17 +92,36 @@ class FixedRetriever(BaseRetriever):
|
|
| 92 |
|
| 93 |
# Question answering function
|
| 94 |
def qa_fetching_repo(url_repo, branch, context_query, prompt_query, chain_type, k):
|
| 95 |
-
|
| 96 |
print("* QA fetching repo")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 97 |
|
| 98 |
-
# load data
|
| 99 |
-
loader = GitLoader(
|
| 100 |
-
clone_url=url_repo,
|
| 101 |
-
repo_path="checkout",
|
| 102 |
-
branch=branch,
|
| 103 |
-
file_filter=lambda file_path: file_path.endswith(".swift") or file_path.endswith(".h") or file_path.endswith(".m")
|
| 104 |
-
)
|
| 105 |
documents = loader.load()
|
|
|
|
| 106 |
|
| 107 |
# split documents into chunks
|
| 108 |
text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
|
|
@@ -182,8 +201,10 @@ def qa_result(_):
|
|
| 182 |
pn.panel("\U0001F916", width=10),
|
| 183 |
pn.Column(
|
| 184 |
result["result"],
|
| 185 |
-
"
|
| 186 |
-
pn.pane.Markdown(
|
|
|
|
|
|
|
| 187 |
)
|
| 188 |
)
|
| 189 |
])
|
|
|
|
| 11 |
from langchain.chat_models import ChatOpenAI
|
| 12 |
from typing import List
|
| 13 |
from langchain.vectorstores import Chroma
|
| 14 |
+
from urllib.parse import urlparse
|
| 15 |
from langchain.docstore.document import Document
|
| 16 |
from langchain.schema import BaseRetriever
|
| 17 |
import panel as pn
|
|
|
|
| 49 |
width=300
|
| 50 |
)
|
| 51 |
|
|
|
|
| 52 |
value="",
|
| 53 |
placeholder="(Optional) Enter your index query here...",
|
| 54 |
height=200,
|
|
|
|
| 92 |
|
| 93 |
# Question answering function
|
| 94 |
def qa_fetching_repo(url_repo, branch, context_query, prompt_query, chain_type, k):
|
|
|
|
| 95 |
print("* QA fetching repo")
|
| 96 |
+
# Extract the repo name from the URL
|
| 97 |
+
url_parsed = urlparse(url_repo)
|
| 98 |
+
name_repo = os.path.basename(url_parsed.path).replace(".git", "")
|
| 99 |
+
|
| 100 |
+
# Check if the repo checkout folder exists and create it if not
|
| 101 |
+
path_repo = os.path.join("checkout", name_repo)
|
| 102 |
+
is_repo_already_checkout = False
|
| 103 |
+
if os.path.exists(path_repo):
|
| 104 |
+
is_repo_already_checkout = True
|
| 105 |
+
else:
|
| 106 |
+
os.makedirs(path_repo)
|
| 107 |
+
|
| 108 |
+
if is_repo_already_checkout:
|
| 109 |
+
loader = GitLoader(
|
| 110 |
+
repo_path=path_repo,
|
| 111 |
+
branch=branch,
|
| 112 |
+
file_filter=lambda file_path: file_path.endswith(".swift") or file_path.endswith(".h") or file_path.endswith(".m")
|
| 113 |
+
)
|
| 114 |
+
else:
|
| 115 |
+
loader = GitLoader(
|
| 116 |
+
clone_url=url_repo,
|
| 117 |
+
repo_path=path_repo,
|
| 118 |
+
branch=branch,
|
| 119 |
+
file_filter=lambda file_path: file_path.endswith(".swift") or file_path.endswith(".h") or file_path.endswith(".m")
|
| 120 |
+
)
|
| 121 |
+
|
| 122 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 123 |
documents = loader.load()
|
| 124 |
+
|
| 125 |
|
| 126 |
# split documents into chunks
|
| 127 |
text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
|
|
|
|
| 201 |
pn.panel("\U0001F916", width=10),
|
| 202 |
pn.Column(
|
| 203 |
result["result"],
|
| 204 |
+
"From the following retrieved sources:",
|
| 205 |
+
pn.pane.Markdown(
|
| 206 |
+
'\n<hr>\n'.join(doc.page_content for doc in result["source_documents"])
|
| 207 |
+
)
|
| 208 |
)
|
| 209 |
)
|
| 210 |
])
|