leftbyte commited on
Commit
f5c7707
·
1 Parent(s): 2a3403b

use DirectoryLoader to pull in local pdf files

Browse files
Files changed (2) hide show
  1. requirements.txt +1 -0
  2. scripts/docsToEmbeddings.py +6 -1
requirements.txt CHANGED
@@ -9,3 +9,4 @@ pinecone-client
9
  pyarrow
10
  tiktoken
11
  typer
 
 
9
  pyarrow
10
  tiktoken
11
  typer
12
+ unstructured
scripts/docsToEmbeddings.py CHANGED
@@ -23,6 +23,7 @@ import pinecone # type: ignore
23
  import tiktoken
24
  from langchain.docstore.document import Document
25
  from langchain.document_loaders import GitbookLoader
 
26
  from langchain.embeddings.base import Embeddings
27
  from langchain.embeddings.openai import OpenAIEmbeddings
28
  from langchain.text_splitter import RecursiveCharacterTextSplitter
@@ -31,7 +32,11 @@ from tiktoken import Encoding
31
 
32
 
33
  def load_local_docs(docs_path: str) -> List[Document]:
34
- pass
 
 
 
 
35
 
36
 
37
  def load_gitbook_docs(docs_url: str) -> List[Document]:
 
23
  import tiktoken
24
  from langchain.docstore.document import Document
25
  from langchain.document_loaders import GitbookLoader
26
+ from langchain.document_loaders import DirectoryLoader
27
  from langchain.embeddings.base import Embeddings
28
  from langchain.embeddings.openai import OpenAIEmbeddings
29
  from langchain.text_splitter import RecursiveCharacterTextSplitter
 
32
 
33
 
34
  def load_local_docs(docs_path: str) -> List[Document]:
35
+ # By default the DirectoryLoader will use the 'Unstructured'
36
+ # Document. Use mode="elements" if we need more metadat in the
37
+ # index.
38
+ loader = DirectoryLoader(docs_path, glob="**/*.pdf")
39
+ return loader.load()
40
 
41
 
42
  def load_gitbook_docs(docs_url: str) -> List[Document]: