Spaces:
Sleeping
Sleeping
parquet argument should be the full path w/ filename
Browse files
scripts/docsToEmbeddings.py
CHANGED
|
@@ -162,9 +162,9 @@ if __name__ == "__main__":
|
|
| 162 |
)
|
| 163 |
parser.add_argument("--openai-api-key", type=str, help="OpenAI API key")
|
| 164 |
parser.add_argument(
|
| 165 |
-
"--output-parquet-
|
| 166 |
type=str,
|
| 167 |
-
help="Path to output parquet
|
| 168 |
)
|
| 169 |
parser.add_argument(
|
| 170 |
"--docs-path", type=str, help="path containing documentation"
|
|
@@ -182,7 +182,7 @@ if __name__ == "__main__":
|
|
| 182 |
pinecone_index_name = args.pinecone_index_name
|
| 183 |
pinecone_environment = args.pinecone_environment
|
| 184 |
openai_api_key = args.openai_api_key
|
| 185 |
-
|
| 186 |
|
| 187 |
openai.api_key = openai_api_key
|
| 188 |
pinecone.init(api_key=pinecone_api_key, environment=pinecone_environment)
|
|
@@ -201,5 +201,5 @@ if __name__ == "__main__":
|
|
| 201 |
embeddings = OpenAIEmbeddingsWrapper(model=embedding_model_name) # type: ignore
|
| 202 |
build_pinecone_index(documents, embeddings, pinecone_index_name)
|
| 203 |
save_dataframe_to_parquet(
|
| 204 |
-
embeddings.document_embedding_dataframe,
|
| 205 |
)
|
|
|
|
| 162 |
)
|
| 163 |
parser.add_argument("--openai-api-key", type=str, help="OpenAI API key")
|
| 164 |
parser.add_argument(
|
| 165 |
+
"--output-parquet-path",
|
| 166 |
type=str,
|
| 167 |
+
help="Path to output parquet file to save the index",
|
| 168 |
)
|
| 169 |
parser.add_argument(
|
| 170 |
"--docs-path", type=str, help="path containing documentation"
|
|
|
|
| 182 |
pinecone_index_name = args.pinecone_index_name
|
| 183 |
pinecone_environment = args.pinecone_environment
|
| 184 |
openai_api_key = args.openai_api_key
|
| 185 |
+
output_parquet_path = args.output_parquet_path
|
| 186 |
|
| 187 |
openai.api_key = openai_api_key
|
| 188 |
pinecone.init(api_key=pinecone_api_key, environment=pinecone_environment)
|
|
|
|
| 201 |
embeddings = OpenAIEmbeddingsWrapper(model=embedding_model_name) # type: ignore
|
| 202 |
build_pinecone_index(documents, embeddings, pinecone_index_name)
|
| 203 |
save_dataframe_to_parquet(
|
| 204 |
+
embeddings.document_embedding_dataframe, output_parquet_path
|
| 205 |
)
|