pro-mls / ingest_data.py
aadishaikh's picture
Upload folder using huggingface_hub
2464276 verified
Raw History Blame
762 Bytes
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.vectorstores.faiss import FAISS
from langchain.embeddings import OpenAIEmbeddings
from langchain_community.document_loaders.mongodb import MongodbLoader
from langchain.document_loaders import UnstructuredFileLoader
import pymongo
print("Loading data...")
loader = UnstructuredFileLoader("state_of_the_union.txt")
raw_documents = loader.load()
print("Splitting text...")
text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100)
documents = text_splitter.split_documents(raw_documents)
print("Creating vectorstore...")
embeddings = OpenAIEmbeddings()
vectorstore = FAISS.from_documents(documents, embeddings)
vectorstore.save_local("faiss_index")