hillbilly-chat / scripts /docsToEmbeddings.py
leftbyte's picture
remove debuggging exit
dfc1199
Raw History Blame Contribute Delete
6.68 kB
#!/usr/bin/env python
"""
Builds a Pinecone index by converting a given set of documents to its embeddings.
To run, you must first create an account with Pinecone and create an
index in the UI with the appropriate embedding dimension (1536 if you
are using text-embedding-ada-002 like this script). You also need an
OpenAI API key. This implementation relies on the fact that the
documentation is hosted with Gitbook. If your documentation does not use
Gitbook, you should use a different document loader.
"""
import argparse
import logging
import sys
from functools import partial
from typing import Dict, List, Optional
import numpy as np
import openai
import pandas as pd
import pinecone # type: ignore
import tiktoken
from langchain.docstore.document import Document
from langchain.document_loaders import GitbookLoader
from langchain.document_loaders import DirectoryLoader
from langchain.embeddings.base import Embeddings
from langchain.embeddings.openai import OpenAIEmbeddings
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.vectorstores import Pinecone
from tiktoken import Encoding
def load_local_docs(docs_path: str) -> List[Document]:
# By default the DirectoryLoader will use the 'Unstructured'
# Document. Use mode="elements" if we need more metadat in the
# index.
loader = DirectoryLoader(docs_path, glob="**/*.pdf")
return loader.load()
def load_gitbook_docs(docs_url: str) -> List[Document]:
"""
Loads documentation from a Gitbook URL.
"""
loader = GitbookLoader(
docs_url,
load_all_paths=True,
)
return loader.load()
def tiktoken_len(text: str, tokenizer: Encoding) -> int:
"""
Returns the number of tokens in a text.
"""
tokens = tokenizer.encode(text, disallowed_special=())
return len(tokens)
def chunk_docs(
documents: List[Document], embedding_model_name: str
) -> List[Document]:
"""
Chunks the documents.
The chunking strategy used in this function is from the following notebook and accompanying
video:
- https://github.com/pinecone-io/examples/blob/master/generation/langchain/handbook/
xx-langchain-chunking.ipynb
- https://www.youtube.com/watch?v=eqOfr4AGLk8
"""
text_splitter = RecursiveCharacterTextSplitter(
chunk_size=400,
chunk_overlap=20,
length_function=partial(
tiktoken_len,
tokenizer=tiktoken.encoding_for_model(embedding_model_name),
),
separators=["\n\n", "\n", " ", ""],
)
return text_splitter.split_documents(documents)
def build_pinecone_index(
documents: List[Document], embeddings: Embeddings, index_name: str
) -> None:
"""
Builds a Pinecone index from a list of documents.
"""
Pinecone.from_documents(
documents, embeddings, index_name=pinecone_index_name
)
# , engine="pyarrow":
def save_dataframe_to_parquet(dataframe: pd.DataFrame, save_path: str) -> None:
"""
Saves a dataframe to parquet.
"""
dataframe.to_parquet(save_path)
class OpenAIEmbeddingsWrapper(OpenAIEmbeddings):
"""
Wrapper around OpenAIEmbeddings that stores the query and document embeddings in memory.
"""
query_text_to_embedding: Dict[str, List[float]] = {}
document_text_to_embedding: Dict[str, List[float]] = {}
def embed_query(self, text: str) -> List[float]:
embedding = super().embed_query(text)
self.query_text_to_embedding[text] = embedding
return embedding
def embed_documents(
self, texts: List[str], chunk_size: Optional[int] = 0
) -> List[List[float]]:
embeddings = super().embed_documents(texts, chunk_size)
for text, embedding in zip(texts, embeddings):
self.document_text_to_embedding[text] = embedding
return embeddings
@property
def query_embedding_dataframe(self) -> pd.DataFrame:
return self._convert_text_to_embedding_map_to_dataframe(
self.query_text_to_embedding
)
@property
def document_embedding_dataframe(self) -> pd.DataFrame:
return self._convert_text_to_embedding_map_to_dataframe(
self.document_text_to_embedding
)
@staticmethod
def _convert_text_to_embedding_map_to_dataframe(
text_to_embedding: Dict[str, List[float]]
) -> pd.DataFrame:
texts, embeddings = map(list, zip(*text_to_embedding.items()))
embedding_arrays = [np.array(embedding) for embedding in embeddings]
return pd.DataFrame.from_dict(
{
"text": texts,
"text_vector": embedding_arrays,
}
)
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO, stream=sys.stdout)
parser = argparse.ArgumentParser()
parser.add_argument("--pinecone-api-key", type=str, help="Pinecone API key")
parser.add_argument(
"--pinecone-index-name", type=str, help="Pinecone index name"
)
parser.add_argument(
"--pinecone-environment", type=str, help="Pinecone environment"
)
parser.add_argument("--openai-api-key", type=str, help="OpenAI API key")
parser.add_argument(
"--output-parquet-path",
type=str,
help="Path to output parquet file to save the index",
)
parser.add_argument(
"--docs-path", type=str, help="path containing documentation"
)
parser.add_argument(
"--docs-url",
type=str,
help="URL containing documentation"
# default="https://docs.arize.com/arize/",
)
args = parser.parse_args()
pinecone_api_key = args.pinecone_api_key
pinecone_index_name = args.pinecone_index_name
pinecone_environment = args.pinecone_environment
openai_api_key = args.openai_api_key
output_parquet_path = args.output_parquet_path
openai.api_key = openai_api_key
pinecone.init(api_key=pinecone_api_key, environment=pinecone_environment)
embedding_model_name = "text-embedding-ada-002"
if args.docs_url is not None:
print(f"docs_url {args.docs_url}")
documents = load_gitbook_docs(args.docs_url)
elif args.docs_path is not None:
documents = load_local_docs(args.docs_path)
else:
assert False, "Either the docs_url or docs_path must be set"
documents = chunk_docs(documents, embedding_model_name=embedding_model_name)
embeddings = OpenAIEmbeddingsWrapper(model=embedding_model_name) # type: ignore
build_pinecone_index(documents, embeddings, pinecone_index_name)
save_dataframe_to_parquet(
embeddings.document_embedding_dataframe, output_parquet_path
)