Spaces:
Sleeping
Sleeping
use DirectoryLoader to pull in local pdf files
Browse files- requirements.txt +1 -0
- scripts/docsToEmbeddings.py +6 -1
requirements.txt
CHANGED
|
@@ -9,3 +9,4 @@ pinecone-client
|
|
| 9 |
pyarrow
|
| 10 |
tiktoken
|
| 11 |
typer
|
|
|
|
|
|
| 9 |
pyarrow
|
| 10 |
tiktoken
|
| 11 |
typer
|
| 12 |
+
unstructured
|
scripts/docsToEmbeddings.py
CHANGED
|
@@ -23,6 +23,7 @@ import pinecone # type: ignore
|
|
| 23 |
import tiktoken
|
| 24 |
from langchain.docstore.document import Document
|
| 25 |
from langchain.document_loaders import GitbookLoader
|
|
|
|
| 26 |
from langchain.embeddings.base import Embeddings
|
| 27 |
from langchain.embeddings.openai import OpenAIEmbeddings
|
| 28 |
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
|
@@ -31,7 +32,11 @@ from tiktoken import Encoding
|
|
| 31 |
|
| 32 |
|
| 33 |
def load_local_docs(docs_path: str) -> List[Document]:
|
| 34 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
|
| 36 |
|
| 37 |
def load_gitbook_docs(docs_url: str) -> List[Document]:
|
|
|
|
| 23 |
import tiktoken
|
| 24 |
from langchain.docstore.document import Document
|
| 25 |
from langchain.document_loaders import GitbookLoader
|
| 26 |
+
from langchain.document_loaders import DirectoryLoader
|
| 27 |
from langchain.embeddings.base import Embeddings
|
| 28 |
from langchain.embeddings.openai import OpenAIEmbeddings
|
| 29 |
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
|
|
|
| 32 |
|
| 33 |
|
| 34 |
def load_local_docs(docs_path: str) -> List[Document]:
|
| 35 |
+
# By default the DirectoryLoader will use the 'Unstructured'
|
| 36 |
+
# Document. Use mode="elements" if we need more metadat in the
|
| 37 |
+
# index.
|
| 38 |
+
loader = DirectoryLoader(docs_path, glob="**/*.pdf")
|
| 39 |
+
return loader.load()
|
| 40 |
|
| 41 |
|
| 42 |
def load_gitbook_docs(docs_url: str) -> List[Document]:
|