Oriol Ganduxé Pregona commited on
Commit
81bba60
·
1 Parent(s): df53c66

Alamofire modifications and cached indexes with 3k chunks

Browse files
checkout/Alamofire ADDED
@@ -0,0 +1 @@
 
 
1
+ Subproject commit fd87f7ce7461197ea5af6e4a865168b8fc913ee0
checkout/BusyNavigationBar ADDED
@@ -0,0 +1 @@
 
 
1
+ Subproject commit 48059ab3bbf4159d71e8ce69c62c51ae284303c7
checkout/tuist ADDED
@@ -0,0 +1 @@
 
 
1
+ Subproject commit 099e3b79347a70822dcb3d10138f50787c4fd0e9
code_qa_app.py CHANGED
@@ -29,7 +29,7 @@ pn.state.template.param.update(
29
  )
30
 
31
  # User inputs
32
- use_cached_repo = pn.widgets.Checkbox(name="Use cached Tuist repo instead")
33
 
34
  url_repo = pn.widgets.TextInput(
35
  value="",
@@ -126,22 +126,26 @@ def qa_fetching_repo(url_repo, branch, context_query, prompt_query, chain_type,
126
 
127
 
128
  # split documents into chunks
129
- text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
130
  texts = text_splitter.split_documents(documents)
131
 
132
  # select which embeddings to use
133
  embeddings = OpenAIEmbeddings()
134
 
 
 
135
  # create the vectorstore to use as the index
136
- db = Chroma.from_documents(texts, embeddings)
 
137
 
138
  return qa(db, context_query, prompt_query, chain_type, k)
139
 
140
  # Question answering function
141
  def qa_cached_tuist(context_query, prompt_query, chain_type, k):
142
 
143
- print("* QA cached tuist")
144
- db = Chroma(persist_directory="tuist_cached_db/", embedding_function=OpenAIEmbeddings())
 
145
 
146
  return qa(db, context_query, prompt_query, chain_type, k)
147
 
@@ -159,7 +163,7 @@ def qa(db, context_query, prompt_query, chain_type, k):
159
  # expose this index in a retriever interface
160
  retriever = db.as_retriever(search_type="similarity", search_kwargs={"k": k})
161
 
162
- # llm=OpenAI()
163
  llm = ChatOpenAI(
164
  model_name = "gpt-3.5-turbo",
165
  temperature=0.7,
 
29
  )
30
 
31
  # User inputs
32
+ use_cached_repo = pn.widgets.Checkbox(name="Use cached Alamofire repo instead")
33
 
34
  url_repo = pn.widgets.TextInput(
35
  value="",
 
126
 
127
 
128
  # split documents into chunks
129
+ text_splitter = RecursiveCharacterTextSplitter(chunk_size=3000, chunk_overlap=0)
130
  texts = text_splitter.split_documents(documents)
131
 
132
  # select which embeddings to use
133
  embeddings = OpenAIEmbeddings()
134
 
135
+ persist_directory = os.path.join("dbs", name_repo)
136
+
137
  # create the vectorstore to use as the index
138
+ db = Chroma.from_documents(texts, embeddings, persist_directory=persist_directory)
139
+ db.persist()
140
 
141
  return qa(db, context_query, prompt_query, chain_type, k)
142
 
143
  # Question answering function
144
  def qa_cached_tuist(context_query, prompt_query, chain_type, k):
145
 
146
+ print("* QA cached Tuist")
147
+ db = Chroma(persist_directory="dbs/Alamofire/", embedding_function=OpenAIEmbeddings())
148
+ # db = Chroma(persist_directory="data/tuist_cached/", embedding_function=OpenAIEmbeddings())
149
 
150
  return qa(db, context_query, prompt_query, chain_type, k)
151
 
 
163
  # expose this index in a retriever interface
164
  retriever = db.as_retriever(search_type="similarity", search_kwargs={"k": k})
165
 
166
+ # llm=OpenAI(temperature=0, model_name="text-davinci-002")
167
  llm = ChatOpenAI(
168
  model_name = "gpt-3.5-turbo",
169
  temperature=0.7,
dbs/Alamofire/chroma-collections.parquet ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ecbe4963d18e41866d6702e52613e682bb31f25ebb4033b8fc9dcc4b2a6f63cb
3
+ size 557
dbs/Alamofire/chroma-embeddings.parquet ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:49b4d111a064634d749d0932a5c8930c09df439179d6eb16a65b5bd57658c991
3
+ size 7730171
dbs/Alamofire/index/id_to_uuid_82e51ffe-b003-42db-a060-7ee52745f81d.pkl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ea43b34fb9f087fd7e508b4e9ebfe6f93aa9ad59b9d2173e6e746043dc86245f
3
+ size 19458
dbs/Alamofire/index/index_82e51ffe-b003-42db-a060-7ee52745f81d.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:63c71563dd9aa8dc6443bf20edcd657bcefd8a4aecdea4613a91d8866308d653
3
+ size 3807056
dbs/Alamofire/index/index_metadata_82e51ffe-b003-42db-a060-7ee52745f81d.pkl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9df5de400e56d4b0c6219e4d17c13166ef51dd2ca574f48cb161a4c7f8d6b30b
3
+ size 74
dbs/Alamofire/index/uuid_to_id_82e51ffe-b003-42db-a060-7ee52745f81d.pkl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ec54074411da3ad2c668c0362ad30c7cca1710832ea3c9423687fc176d68be66
3
+ size 22751
{tuist_cached_db → dbs/tuist_cached}/chroma-collections.parquet RENAMED
File without changes
{tuist_cached_db → dbs/tuist_cached}/chroma-embeddings.parquet RENAMED
File without changes
{tuist_cached_db → dbs/tuist_cached}/index/id_to_uuid_dbb9dc64-2123-4c5d-a69d-a08a9d0b3485.pkl RENAMED
File without changes
{tuist_cached_db → dbs/tuist_cached}/index/index_dbb9dc64-2123-4c5d-a69d-a08a9d0b3485.bin RENAMED
File without changes
{tuist_cached_db → dbs/tuist_cached}/index/index_metadata_dbb9dc64-2123-4c5d-a69d-a08a9d0b3485.pkl RENAMED
File without changes
{tuist_cached_db → dbs/tuist_cached}/index/uuid_to_id_dbb9dc64-2123-4c5d-a69d-a08a9d0b3485.pkl RENAMED
File without changes