File size: 6,679 Bytes
bf7ddae
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
f5c7707
bf7ddae
 
 
 
 
 
 
 
f5c7707
 
 
 
 
bf7ddae
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2a3403b
bf7ddae
2a3403b
bf7ddae
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2a3403b
bf7ddae
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2a3403b
bf7ddae
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
#!/usr/bin/env python
"""
Builds a Pinecone index by converting a given set of documents to its embeddings.

To run, you must first create an account with Pinecone and create an
index in the UI with the appropriate embedding dimension (1536 if you
are using text-embedding-ada-002 like this script).  You also need an
OpenAI API key. This implementation relies on the fact that the
documentation is hosted with Gitbook. If your documentation does not use
Gitbook, you should use a different document loader.
"""

import argparse
import logging
import sys
from functools import partial
from typing import Dict, List, Optional

import numpy as np
import openai
import pandas as pd
import pinecone  # type: ignore
import tiktoken
from langchain.docstore.document import Document
from langchain.document_loaders import GitbookLoader
from langchain.document_loaders import DirectoryLoader
from langchain.embeddings.base import Embeddings
from langchain.embeddings.openai import OpenAIEmbeddings
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.vectorstores import Pinecone
from tiktoken import Encoding


def load_local_docs(docs_path: str) -> List[Document]:
    # By default the DirectoryLoader will use the 'Unstructured'
    # Document. Use mode="elements" if we need more metadat in the
    # index.
    loader = DirectoryLoader(docs_path, glob="**/*.pdf")
    return loader.load()


def load_gitbook_docs(docs_url: str) -> List[Document]:
    """
    Loads documentation from a Gitbook URL.
    """

    loader = GitbookLoader(
        docs_url,
        load_all_paths=True,
    )
    return loader.load()


def tiktoken_len(text: str, tokenizer: Encoding) -> int:
    """
    Returns the number of tokens in a text.
    """

    tokens = tokenizer.encode(text, disallowed_special=())
    return len(tokens)


def chunk_docs(
    documents: List[Document], embedding_model_name: str
) -> List[Document]:
    """
    Chunks the documents.

    The chunking strategy used in this function is from the following notebook and accompanying
    video:

    - https://github.com/pinecone-io/examples/blob/master/generation/langchain/handbook/
      xx-langchain-chunking.ipynb
    - https://www.youtube.com/watch?v=eqOfr4AGLk8
    """

    text_splitter = RecursiveCharacterTextSplitter(
        chunk_size=400,
        chunk_overlap=20,
        length_function=partial(
            tiktoken_len,
            tokenizer=tiktoken.encoding_for_model(embedding_model_name),
        ),
        separators=["\n\n", "\n", " ", ""],
    )
    return text_splitter.split_documents(documents)


def build_pinecone_index(
    documents: List[Document], embeddings: Embeddings, index_name: str
) -> None:
    """
    Builds a Pinecone index from a list of documents.
    """

    Pinecone.from_documents(
        documents, embeddings, index_name=pinecone_index_name
    )


# , engine="pyarrow":
def save_dataframe_to_parquet(dataframe: pd.DataFrame, save_path: str) -> None:
    """
    Saves a dataframe to parquet.
    """

    dataframe.to_parquet(save_path)


class OpenAIEmbeddingsWrapper(OpenAIEmbeddings):
    """
    Wrapper around OpenAIEmbeddings that stores the query and document embeddings in memory.
    """

    query_text_to_embedding: Dict[str, List[float]] = {}
    document_text_to_embedding: Dict[str, List[float]] = {}

    def embed_query(self, text: str) -> List[float]:
        embedding = super().embed_query(text)
        self.query_text_to_embedding[text] = embedding
        return embedding

    def embed_documents(
        self, texts: List[str], chunk_size: Optional[int] = 0
    ) -> List[List[float]]:
        embeddings = super().embed_documents(texts, chunk_size)
        for text, embedding in zip(texts, embeddings):
            self.document_text_to_embedding[text] = embedding
        return embeddings

    @property
    def query_embedding_dataframe(self) -> pd.DataFrame:
        return self._convert_text_to_embedding_map_to_dataframe(
            self.query_text_to_embedding
        )

    @property
    def document_embedding_dataframe(self) -> pd.DataFrame:
        return self._convert_text_to_embedding_map_to_dataframe(
            self.document_text_to_embedding
        )

    @staticmethod
    def _convert_text_to_embedding_map_to_dataframe(
        text_to_embedding: Dict[str, List[float]]
    ) -> pd.DataFrame:
        texts, embeddings = map(list, zip(*text_to_embedding.items()))
        embedding_arrays = [np.array(embedding) for embedding in embeddings]
        return pd.DataFrame.from_dict(
            {
                "text": texts,
                "text_vector": embedding_arrays,
            }
        )


if __name__ == "__main__":
    logging.basicConfig(level=logging.INFO, stream=sys.stdout)

    parser = argparse.ArgumentParser()
    parser.add_argument("--pinecone-api-key", type=str, help="Pinecone API key")
    parser.add_argument(
        "--pinecone-index-name", type=str, help="Pinecone index name"
    )
    parser.add_argument(
        "--pinecone-environment", type=str, help="Pinecone environment"
    )
    parser.add_argument("--openai-api-key", type=str, help="OpenAI API key")
    parser.add_argument(
        "--output-parquet-path",
        type=str,
        help="Path to output parquet file to save the index",
    )
    parser.add_argument(
        "--docs-path", type=str, help="path containing documentation"
    )
    parser.add_argument(
        "--docs-url",
        type=str,
        help="URL containing documentation"
        # default="https://docs.arize.com/arize/",
    )

    args = parser.parse_args()

    pinecone_api_key = args.pinecone_api_key
    pinecone_index_name = args.pinecone_index_name
    pinecone_environment = args.pinecone_environment
    openai_api_key = args.openai_api_key
    output_parquet_path = args.output_parquet_path

    openai.api_key = openai_api_key
    pinecone.init(api_key=pinecone_api_key, environment=pinecone_environment)
    embedding_model_name = "text-embedding-ada-002"

    if args.docs_url is not None:
        print(f"docs_url {args.docs_url}")
        documents = load_gitbook_docs(args.docs_url)
    elif args.docs_path is not None:
        documents = load_local_docs(args.docs_path)
    else:
        assert False, "Either the docs_url or docs_path must be set"

    documents = chunk_docs(documents, embedding_model_name=embedding_model_name)
    embeddings = OpenAIEmbeddingsWrapper(model=embedding_model_name)  # type: ignore
    build_pinecone_index(documents, embeddings, pinecone_index_name)
    save_dataframe_to_parquet(
        embeddings.document_embedding_dataframe, output_parquet_path
    )