Oriol Ganduxé Pregona commited on
Commit ·
81bba60
1
Parent(s): df53c66
Alamofire modifications and cached indexes with 3k chunks
Browse files- checkout/Alamofire +1 -0
- checkout/BusyNavigationBar +1 -0
- checkout/tuist +1 -0
- code_qa_app.py +10 -6
- dbs/Alamofire/chroma-collections.parquet +3 -0
- dbs/Alamofire/chroma-embeddings.parquet +3 -0
- dbs/Alamofire/index/id_to_uuid_82e51ffe-b003-42db-a060-7ee52745f81d.pkl +3 -0
- dbs/Alamofire/index/index_82e51ffe-b003-42db-a060-7ee52745f81d.bin +3 -0
- dbs/Alamofire/index/index_metadata_82e51ffe-b003-42db-a060-7ee52745f81d.pkl +3 -0
- dbs/Alamofire/index/uuid_to_id_82e51ffe-b003-42db-a060-7ee52745f81d.pkl +3 -0
- {tuist_cached_db → dbs/tuist_cached}/chroma-collections.parquet +0 -0
- {tuist_cached_db → dbs/tuist_cached}/chroma-embeddings.parquet +0 -0
- {tuist_cached_db → dbs/tuist_cached}/index/id_to_uuid_dbb9dc64-2123-4c5d-a69d-a08a9d0b3485.pkl +0 -0
- {tuist_cached_db → dbs/tuist_cached}/index/index_dbb9dc64-2123-4c5d-a69d-a08a9d0b3485.bin +0 -0
- {tuist_cached_db → dbs/tuist_cached}/index/index_metadata_dbb9dc64-2123-4c5d-a69d-a08a9d0b3485.pkl +0 -0
- {tuist_cached_db → dbs/tuist_cached}/index/uuid_to_id_dbb9dc64-2123-4c5d-a69d-a08a9d0b3485.pkl +0 -0
checkout/Alamofire
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
Subproject commit fd87f7ce7461197ea5af6e4a865168b8fc913ee0
|
checkout/BusyNavigationBar
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
Subproject commit 48059ab3bbf4159d71e8ce69c62c51ae284303c7
|
checkout/tuist
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
Subproject commit 099e3b79347a70822dcb3d10138f50787c4fd0e9
|
code_qa_app.py
CHANGED
|
@@ -29,7 +29,7 @@ pn.state.template.param.update(
|
|
| 29 |
)
|
| 30 |
|
| 31 |
# User inputs
|
| 32 |
-
use_cached_repo = pn.widgets.Checkbox(name="Use cached
|
| 33 |
|
| 34 |
url_repo = pn.widgets.TextInput(
|
| 35 |
value="",
|
|
@@ -126,22 +126,26 @@ def qa_fetching_repo(url_repo, branch, context_query, prompt_query, chain_type,
|
|
| 126 |
|
| 127 |
|
| 128 |
# split documents into chunks
|
| 129 |
-
text_splitter = RecursiveCharacterTextSplitter(chunk_size=
|
| 130 |
texts = text_splitter.split_documents(documents)
|
| 131 |
|
| 132 |
# select which embeddings to use
|
| 133 |
embeddings = OpenAIEmbeddings()
|
| 134 |
|
|
|
|
|
|
|
| 135 |
# create the vectorstore to use as the index
|
| 136 |
-
db = Chroma.from_documents(texts, embeddings)
|
|
|
|
| 137 |
|
| 138 |
return qa(db, context_query, prompt_query, chain_type, k)
|
| 139 |
|
| 140 |
# Question answering function
|
| 141 |
def qa_cached_tuist(context_query, prompt_query, chain_type, k):
|
| 142 |
|
| 143 |
-
print("* QA cached
|
| 144 |
-
db = Chroma(persist_directory="
|
|
|
|
| 145 |
|
| 146 |
return qa(db, context_query, prompt_query, chain_type, k)
|
| 147 |
|
|
@@ -159,7 +163,7 @@ def qa(db, context_query, prompt_query, chain_type, k):
|
|
| 159 |
# expose this index in a retriever interface
|
| 160 |
retriever = db.as_retriever(search_type="similarity", search_kwargs={"k": k})
|
| 161 |
|
| 162 |
-
# llm=OpenAI()
|
| 163 |
llm = ChatOpenAI(
|
| 164 |
model_name = "gpt-3.5-turbo",
|
| 165 |
temperature=0.7,
|
|
|
|
| 29 |
)
|
| 30 |
|
| 31 |
# User inputs
|
| 32 |
+
use_cached_repo = pn.widgets.Checkbox(name="Use cached Alamofire repo instead")
|
| 33 |
|
| 34 |
url_repo = pn.widgets.TextInput(
|
| 35 |
value="",
|
|
|
|
| 126 |
|
| 127 |
|
| 128 |
# split documents into chunks
|
| 129 |
+
text_splitter = RecursiveCharacterTextSplitter(chunk_size=3000, chunk_overlap=0)
|
| 130 |
texts = text_splitter.split_documents(documents)
|
| 131 |
|
| 132 |
# select which embeddings to use
|
| 133 |
embeddings = OpenAIEmbeddings()
|
| 134 |
|
| 135 |
+
persist_directory = os.path.join("dbs", name_repo)
|
| 136 |
+
|
| 137 |
# create the vectorstore to use as the index
|
| 138 |
+
db = Chroma.from_documents(texts, embeddings, persist_directory=persist_directory)
|
| 139 |
+
db.persist()
|
| 140 |
|
| 141 |
return qa(db, context_query, prompt_query, chain_type, k)
|
| 142 |
|
| 143 |
# Question answering function
|
| 144 |
def qa_cached_tuist(context_query, prompt_query, chain_type, k):
|
| 145 |
|
| 146 |
+
print("* QA cached Tuist")
|
| 147 |
+
db = Chroma(persist_directory="dbs/Alamofire/", embedding_function=OpenAIEmbeddings())
|
| 148 |
+
# db = Chroma(persist_directory="data/tuist_cached/", embedding_function=OpenAIEmbeddings())
|
| 149 |
|
| 150 |
return qa(db, context_query, prompt_query, chain_type, k)
|
| 151 |
|
|
|
|
| 163 |
# expose this index in a retriever interface
|
| 164 |
retriever = db.as_retriever(search_type="similarity", search_kwargs={"k": k})
|
| 165 |
|
| 166 |
+
# llm=OpenAI(temperature=0, model_name="text-davinci-002")
|
| 167 |
llm = ChatOpenAI(
|
| 168 |
model_name = "gpt-3.5-turbo",
|
| 169 |
temperature=0.7,
|
dbs/Alamofire/chroma-collections.parquet
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ecbe4963d18e41866d6702e52613e682bb31f25ebb4033b8fc9dcc4b2a6f63cb
|
| 3 |
+
size 557
|
dbs/Alamofire/chroma-embeddings.parquet
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:49b4d111a064634d749d0932a5c8930c09df439179d6eb16a65b5bd57658c991
|
| 3 |
+
size 7730171
|
dbs/Alamofire/index/id_to_uuid_82e51ffe-b003-42db-a060-7ee52745f81d.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ea43b34fb9f087fd7e508b4e9ebfe6f93aa9ad59b9d2173e6e746043dc86245f
|
| 3 |
+
size 19458
|
dbs/Alamofire/index/index_82e51ffe-b003-42db-a060-7ee52745f81d.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:63c71563dd9aa8dc6443bf20edcd657bcefd8a4aecdea4613a91d8866308d653
|
| 3 |
+
size 3807056
|
dbs/Alamofire/index/index_metadata_82e51ffe-b003-42db-a060-7ee52745f81d.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9df5de400e56d4b0c6219e4d17c13166ef51dd2ca574f48cb161a4c7f8d6b30b
|
| 3 |
+
size 74
|
dbs/Alamofire/index/uuid_to_id_82e51ffe-b003-42db-a060-7ee52745f81d.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ec54074411da3ad2c668c0362ad30c7cca1710832ea3c9423687fc176d68be66
|
| 3 |
+
size 22751
|
{tuist_cached_db → dbs/tuist_cached}/chroma-collections.parquet
RENAMED
|
File without changes
|
{tuist_cached_db → dbs/tuist_cached}/chroma-embeddings.parquet
RENAMED
|
File without changes
|
{tuist_cached_db → dbs/tuist_cached}/index/id_to_uuid_dbb9dc64-2123-4c5d-a69d-a08a9d0b3485.pkl
RENAMED
|
File without changes
|
{tuist_cached_db → dbs/tuist_cached}/index/index_dbb9dc64-2123-4c5d-a69d-a08a9d0b3485.bin
RENAMED
|
File without changes
|
{tuist_cached_db → dbs/tuist_cached}/index/index_metadata_dbb9dc64-2123-4c5d-a69d-a08a9d0b3485.pkl
RENAMED
|
File without changes
|
{tuist_cached_db → dbs/tuist_cached}/index/uuid_to_id_dbb9dc64-2123-4c5d-a69d-a08a9d0b3485.pkl
RENAMED
|
File without changes
|