Spaces:
Sleeping
Sleeping
File size: 6,679 Bytes
bf7ddae f5c7707 bf7ddae f5c7707 bf7ddae 2a3403b bf7ddae 2a3403b bf7ddae 2a3403b bf7ddae 2a3403b bf7ddae | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 | #!/usr/bin/env python
"""
Builds a Pinecone index by converting a given set of documents to its embeddings.
To run, you must first create an account with Pinecone and create an
index in the UI with the appropriate embedding dimension (1536 if you
are using text-embedding-ada-002 like this script). You also need an
OpenAI API key. This implementation relies on the fact that the
documentation is hosted with Gitbook. If your documentation does not use
Gitbook, you should use a different document loader.
"""
import argparse
import logging
import sys
from functools import partial
from typing import Dict, List, Optional
import numpy as np
import openai
import pandas as pd
import pinecone # type: ignore
import tiktoken
from langchain.docstore.document import Document
from langchain.document_loaders import GitbookLoader
from langchain.document_loaders import DirectoryLoader
from langchain.embeddings.base import Embeddings
from langchain.embeddings.openai import OpenAIEmbeddings
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.vectorstores import Pinecone
from tiktoken import Encoding
def load_local_docs(docs_path: str) -> List[Document]:
# By default the DirectoryLoader will use the 'Unstructured'
# Document. Use mode="elements" if we need more metadat in the
# index.
loader = DirectoryLoader(docs_path, glob="**/*.pdf")
return loader.load()
def load_gitbook_docs(docs_url: str) -> List[Document]:
"""
Loads documentation from a Gitbook URL.
"""
loader = GitbookLoader(
docs_url,
load_all_paths=True,
)
return loader.load()
def tiktoken_len(text: str, tokenizer: Encoding) -> int:
"""
Returns the number of tokens in a text.
"""
tokens = tokenizer.encode(text, disallowed_special=())
return len(tokens)
def chunk_docs(
documents: List[Document], embedding_model_name: str
) -> List[Document]:
"""
Chunks the documents.
The chunking strategy used in this function is from the following notebook and accompanying
video:
- https://github.com/pinecone-io/examples/blob/master/generation/langchain/handbook/
xx-langchain-chunking.ipynb
- https://www.youtube.com/watch?v=eqOfr4AGLk8
"""
text_splitter = RecursiveCharacterTextSplitter(
chunk_size=400,
chunk_overlap=20,
length_function=partial(
tiktoken_len,
tokenizer=tiktoken.encoding_for_model(embedding_model_name),
),
separators=["\n\n", "\n", " ", ""],
)
return text_splitter.split_documents(documents)
def build_pinecone_index(
documents: List[Document], embeddings: Embeddings, index_name: str
) -> None:
"""
Builds a Pinecone index from a list of documents.
"""
Pinecone.from_documents(
documents, embeddings, index_name=pinecone_index_name
)
# , engine="pyarrow":
def save_dataframe_to_parquet(dataframe: pd.DataFrame, save_path: str) -> None:
"""
Saves a dataframe to parquet.
"""
dataframe.to_parquet(save_path)
class OpenAIEmbeddingsWrapper(OpenAIEmbeddings):
"""
Wrapper around OpenAIEmbeddings that stores the query and document embeddings in memory.
"""
query_text_to_embedding: Dict[str, List[float]] = {}
document_text_to_embedding: Dict[str, List[float]] = {}
def embed_query(self, text: str) -> List[float]:
embedding = super().embed_query(text)
self.query_text_to_embedding[text] = embedding
return embedding
def embed_documents(
self, texts: List[str], chunk_size: Optional[int] = 0
) -> List[List[float]]:
embeddings = super().embed_documents(texts, chunk_size)
for text, embedding in zip(texts, embeddings):
self.document_text_to_embedding[text] = embedding
return embeddings
@property
def query_embedding_dataframe(self) -> pd.DataFrame:
return self._convert_text_to_embedding_map_to_dataframe(
self.query_text_to_embedding
)
@property
def document_embedding_dataframe(self) -> pd.DataFrame:
return self._convert_text_to_embedding_map_to_dataframe(
self.document_text_to_embedding
)
@staticmethod
def _convert_text_to_embedding_map_to_dataframe(
text_to_embedding: Dict[str, List[float]]
) -> pd.DataFrame:
texts, embeddings = map(list, zip(*text_to_embedding.items()))
embedding_arrays = [np.array(embedding) for embedding in embeddings]
return pd.DataFrame.from_dict(
{
"text": texts,
"text_vector": embedding_arrays,
}
)
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO, stream=sys.stdout)
parser = argparse.ArgumentParser()
parser.add_argument("--pinecone-api-key", type=str, help="Pinecone API key")
parser.add_argument(
"--pinecone-index-name", type=str, help="Pinecone index name"
)
parser.add_argument(
"--pinecone-environment", type=str, help="Pinecone environment"
)
parser.add_argument("--openai-api-key", type=str, help="OpenAI API key")
parser.add_argument(
"--output-parquet-path",
type=str,
help="Path to output parquet file to save the index",
)
parser.add_argument(
"--docs-path", type=str, help="path containing documentation"
)
parser.add_argument(
"--docs-url",
type=str,
help="URL containing documentation"
# default="https://docs.arize.com/arize/",
)
args = parser.parse_args()
pinecone_api_key = args.pinecone_api_key
pinecone_index_name = args.pinecone_index_name
pinecone_environment = args.pinecone_environment
openai_api_key = args.openai_api_key
output_parquet_path = args.output_parquet_path
openai.api_key = openai_api_key
pinecone.init(api_key=pinecone_api_key, environment=pinecone_environment)
embedding_model_name = "text-embedding-ada-002"
if args.docs_url is not None:
print(f"docs_url {args.docs_url}")
documents = load_gitbook_docs(args.docs_url)
elif args.docs_path is not None:
documents = load_local_docs(args.docs_path)
else:
assert False, "Either the docs_url or docs_path must be set"
documents = chunk_docs(documents, embedding_model_name=embedding_model_name)
embeddings = OpenAIEmbeddingsWrapper(model=embedding_model_name) # type: ignore
build_pinecone_index(documents, embeddings, pinecone_index_name)
save_dataframe_to_parquet(
embeddings.document_embedding_dataframe, output_parquet_path
)
|