leftbyte commited on
Commit
f04d05e
·
unverified ·
2 Parent(s): ad82e2ddfc1199

Merge pull request #1 from leftbyte/dphung/build_pinecone_index

Browse files
Files changed (2) hide show
  1. requirements.txt +8 -0
  2. scripts/docsToEmbeddings.py +209 -0
requirements.txt CHANGED
@@ -1,4 +1,12 @@
1
  black
 
 
2
  gradio
 
3
  openai
 
 
 
 
4
  typer
 
 
1
  black
2
+ bs4
3
+ fastparquet
4
  gradio
5
+ langchain
6
  openai
7
+ pandas
8
+ pinecone-client
9
+ pyarrow
10
+ tiktoken
11
  typer
12
+ unstructured
scripts/docsToEmbeddings.py ADDED
@@ -0,0 +1,209 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python
2
+ """
3
+ Builds a Pinecone index by converting a given set of documents to its embeddings.
4
+
5
+ To run, you must first create an account with Pinecone and create an
6
+ index in the UI with the appropriate embedding dimension (1536 if you
7
+ are using text-embedding-ada-002 like this script). You also need an
8
+ OpenAI API key. This implementation relies on the fact that the
9
+ documentation is hosted with Gitbook. If your documentation does not use
10
+ Gitbook, you should use a different document loader.
11
+ """
12
+
13
+ import argparse
14
+ import logging
15
+ import sys
16
+ from functools import partial
17
+ from typing import Dict, List, Optional
18
+
19
+ import numpy as np
20
+ import openai
21
+ import pandas as pd
22
+ import pinecone # type: ignore
23
+ import tiktoken
24
+ from langchain.docstore.document import Document
25
+ from langchain.document_loaders import GitbookLoader
26
+ from langchain.document_loaders import DirectoryLoader
27
+ from langchain.embeddings.base import Embeddings
28
+ from langchain.embeddings.openai import OpenAIEmbeddings
29
+ from langchain.text_splitter import RecursiveCharacterTextSplitter
30
+ from langchain.vectorstores import Pinecone
31
+ from tiktoken import Encoding
32
+
33
+
34
+ def load_local_docs(docs_path: str) -> List[Document]:
35
+ # By default the DirectoryLoader will use the 'Unstructured'
36
+ # Document. Use mode="elements" if we need more metadat in the
37
+ # index.
38
+ loader = DirectoryLoader(docs_path, glob="**/*.pdf")
39
+ return loader.load()
40
+
41
+
42
+ def load_gitbook_docs(docs_url: str) -> List[Document]:
43
+ """
44
+ Loads documentation from a Gitbook URL.
45
+ """
46
+
47
+ loader = GitbookLoader(
48
+ docs_url,
49
+ load_all_paths=True,
50
+ )
51
+ return loader.load()
52
+
53
+
54
+ def tiktoken_len(text: str, tokenizer: Encoding) -> int:
55
+ """
56
+ Returns the number of tokens in a text.
57
+ """
58
+
59
+ tokens = tokenizer.encode(text, disallowed_special=())
60
+ return len(tokens)
61
+
62
+
63
+ def chunk_docs(
64
+ documents: List[Document], embedding_model_name: str
65
+ ) -> List[Document]:
66
+ """
67
+ Chunks the documents.
68
+
69
+ The chunking strategy used in this function is from the following notebook and accompanying
70
+ video:
71
+
72
+ - https://github.com/pinecone-io/examples/blob/master/generation/langchain/handbook/
73
+ xx-langchain-chunking.ipynb
74
+ - https://www.youtube.com/watch?v=eqOfr4AGLk8
75
+ """
76
+
77
+ text_splitter = RecursiveCharacterTextSplitter(
78
+ chunk_size=400,
79
+ chunk_overlap=20,
80
+ length_function=partial(
81
+ tiktoken_len,
82
+ tokenizer=tiktoken.encoding_for_model(embedding_model_name),
83
+ ),
84
+ separators=["\n\n", "\n", " ", ""],
85
+ )
86
+ return text_splitter.split_documents(documents)
87
+
88
+
89
+ def build_pinecone_index(
90
+ documents: List[Document], embeddings: Embeddings, index_name: str
91
+ ) -> None:
92
+ """
93
+ Builds a Pinecone index from a list of documents.
94
+ """
95
+
96
+ Pinecone.from_documents(
97
+ documents, embeddings, index_name=pinecone_index_name
98
+ )
99
+
100
+
101
+ # , engine="pyarrow":
102
+ def save_dataframe_to_parquet(dataframe: pd.DataFrame, save_path: str) -> None:
103
+ """
104
+ Saves a dataframe to parquet.
105
+ """
106
+
107
+ dataframe.to_parquet(save_path)
108
+
109
+
110
+ class OpenAIEmbeddingsWrapper(OpenAIEmbeddings):
111
+ """
112
+ Wrapper around OpenAIEmbeddings that stores the query and document embeddings in memory.
113
+ """
114
+
115
+ query_text_to_embedding: Dict[str, List[float]] = {}
116
+ document_text_to_embedding: Dict[str, List[float]] = {}
117
+
118
+ def embed_query(self, text: str) -> List[float]:
119
+ embedding = super().embed_query(text)
120
+ self.query_text_to_embedding[text] = embedding
121
+ return embedding
122
+
123
+ def embed_documents(
124
+ self, texts: List[str], chunk_size: Optional[int] = 0
125
+ ) -> List[List[float]]:
126
+ embeddings = super().embed_documents(texts, chunk_size)
127
+ for text, embedding in zip(texts, embeddings):
128
+ self.document_text_to_embedding[text] = embedding
129
+ return embeddings
130
+
131
+ @property
132
+ def query_embedding_dataframe(self) -> pd.DataFrame:
133
+ return self._convert_text_to_embedding_map_to_dataframe(
134
+ self.query_text_to_embedding
135
+ )
136
+
137
+ @property
138
+ def document_embedding_dataframe(self) -> pd.DataFrame:
139
+ return self._convert_text_to_embedding_map_to_dataframe(
140
+ self.document_text_to_embedding
141
+ )
142
+
143
+ @staticmethod
144
+ def _convert_text_to_embedding_map_to_dataframe(
145
+ text_to_embedding: Dict[str, List[float]]
146
+ ) -> pd.DataFrame:
147
+ texts, embeddings = map(list, zip(*text_to_embedding.items()))
148
+ embedding_arrays = [np.array(embedding) for embedding in embeddings]
149
+ return pd.DataFrame.from_dict(
150
+ {
151
+ "text": texts,
152
+ "text_vector": embedding_arrays,
153
+ }
154
+ )
155
+
156
+
157
+ if __name__ == "__main__":
158
+ logging.basicConfig(level=logging.INFO, stream=sys.stdout)
159
+
160
+ parser = argparse.ArgumentParser()
161
+ parser.add_argument("--pinecone-api-key", type=str, help="Pinecone API key")
162
+ parser.add_argument(
163
+ "--pinecone-index-name", type=str, help="Pinecone index name"
164
+ )
165
+ parser.add_argument(
166
+ "--pinecone-environment", type=str, help="Pinecone environment"
167
+ )
168
+ parser.add_argument("--openai-api-key", type=str, help="OpenAI API key")
169
+ parser.add_argument(
170
+ "--output-parquet-path",
171
+ type=str,
172
+ help="Path to output parquet file to save the index",
173
+ )
174
+ parser.add_argument(
175
+ "--docs-path", type=str, help="path containing documentation"
176
+ )
177
+ parser.add_argument(
178
+ "--docs-url",
179
+ type=str,
180
+ help="URL containing documentation"
181
+ # default="https://docs.arize.com/arize/",
182
+ )
183
+
184
+ args = parser.parse_args()
185
+
186
+ pinecone_api_key = args.pinecone_api_key
187
+ pinecone_index_name = args.pinecone_index_name
188
+ pinecone_environment = args.pinecone_environment
189
+ openai_api_key = args.openai_api_key
190
+ output_parquet_path = args.output_parquet_path
191
+
192
+ openai.api_key = openai_api_key
193
+ pinecone.init(api_key=pinecone_api_key, environment=pinecone_environment)
194
+ embedding_model_name = "text-embedding-ada-002"
195
+
196
+ if args.docs_url is not None:
197
+ print(f"docs_url {args.docs_url}")
198
+ documents = load_gitbook_docs(args.docs_url)
199
+ elif args.docs_path is not None:
200
+ documents = load_local_docs(args.docs_path)
201
+ else:
202
+ assert False, "Either the docs_url or docs_path must be set"
203
+
204
+ documents = chunk_docs(documents, embedding_model_name=embedding_model_name)
205
+ embeddings = OpenAIEmbeddingsWrapper(model=embedding_model_name) # type: ignore
206
+ build_pinecone_index(documents, embeddings, pinecone_index_name)
207
+ save_dataframe_to_parquet(
208
+ embeddings.document_embedding_dataframe, output_parquet_path
209
+ )