Stéphane G commited on
Commit
040ae1b
·
1 Parent(s): 04a5875

add logic to checkout repos in subfolders, and avoid cloning if already checked out

Browse files
Files changed (1) hide show
  1. code_qa_app.py +32 -11
code_qa_app.py CHANGED
@@ -11,6 +11,7 @@ from langchain.vectorstores import Chroma
11
  from langchain.chat_models import ChatOpenAI
12
  from typing import List
13
  from langchain.vectorstores import Chroma
 
14
  from langchain.docstore.document import Document
15
  from langchain.schema import BaseRetriever
16
  import panel as pn
@@ -48,7 +49,6 @@ openaikey = pn.widgets.PasswordInput(
48
  width=300
49
  )
50
 
51
- context_texteditor = pn.widgets.TextEditor(
52
  value="",
53
  placeholder="(Optional) Enter your index query here...",
54
  height=200,
@@ -92,17 +92,36 @@ class FixedRetriever(BaseRetriever):
92
 
93
  # Question answering function
94
  def qa_fetching_repo(url_repo, branch, context_query, prompt_query, chain_type, k):
95
-
96
  print("* QA fetching repo")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
97
 
98
- # load data
99
- loader = GitLoader(
100
- clone_url=url_repo,
101
- repo_path="checkout",
102
- branch=branch,
103
- file_filter=lambda file_path: file_path.endswith(".swift") or file_path.endswith(".h") or file_path.endswith(".m")
104
- )
105
  documents = loader.load()
 
106
 
107
  # split documents into chunks
108
  text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
@@ -182,8 +201,10 @@ def qa_result(_):
182
  pn.panel("\U0001F916", width=10),
183
  pn.Column(
184
  result["result"],
185
- "Relevant source text:",
186
- pn.pane.Markdown('\n--------------------------------------------------------------------\n'.join(doc.page_content for doc in result["source_documents"]))
 
 
187
  )
188
  )
189
  ])
 
11
  from langchain.chat_models import ChatOpenAI
12
  from typing import List
13
  from langchain.vectorstores import Chroma
14
+ from urllib.parse import urlparse
15
  from langchain.docstore.document import Document
16
  from langchain.schema import BaseRetriever
17
  import panel as pn
 
49
  width=300
50
  )
51
 
 
52
  value="",
53
  placeholder="(Optional) Enter your index query here...",
54
  height=200,
 
92
 
93
  # Question answering function
94
  def qa_fetching_repo(url_repo, branch, context_query, prompt_query, chain_type, k):
 
95
  print("* QA fetching repo")
96
+ # Extract the repo name from the URL
97
+ url_parsed = urlparse(url_repo)
98
+ name_repo = os.path.basename(url_parsed.path).replace(".git", "")
99
+
100
+ # Check if the repo checkout folder exists and create it if not
101
+ path_repo = os.path.join("checkout", name_repo)
102
+ is_repo_already_checkout = False
103
+ if os.path.exists(path_repo):
104
+ is_repo_already_checkout = True
105
+ else:
106
+ os.makedirs(path_repo)
107
+
108
+ if is_repo_already_checkout:
109
+ loader = GitLoader(
110
+ repo_path=path_repo,
111
+ branch=branch,
112
+ file_filter=lambda file_path: file_path.endswith(".swift") or file_path.endswith(".h") or file_path.endswith(".m")
113
+ )
114
+ else:
115
+ loader = GitLoader(
116
+ clone_url=url_repo,
117
+ repo_path=path_repo,
118
+ branch=branch,
119
+ file_filter=lambda file_path: file_path.endswith(".swift") or file_path.endswith(".h") or file_path.endswith(".m")
120
+ )
121
+
122
 
 
 
 
 
 
 
 
123
  documents = loader.load()
124
+
125
 
126
  # split documents into chunks
127
  text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
 
201
  pn.panel("\U0001F916", width=10),
202
  pn.Column(
203
  result["result"],
204
+ "From the following retrieved sources:",
205
+ pn.pane.Markdown(
206
+ '\n<hr>\n'.join(doc.page_content for doc in result["source_documents"])
207
+ )
208
  )
209
  )
210
  ])