Spaces:
Runtime error
Runtime error
Download ingest_data.py from tafsol/pro-mls: direct link, hf CLI and curl.
- Browser
- Download file 762 Bytes
-
https://huggingface.co/spaces/tafsol/pro-mls/resolve/main/ingest_data.py
- Command line
-
hf download hf://spaces/tafsol/pro-mls/ingest_data.py
-
curl -L -o ingest_data.py https://huggingface.co/spaces/tafsol/pro-mls/resolve/main/ingest_data.py
762 Bytes
| from langchain.text_splitter import RecursiveCharacterTextSplitter | |
| from langchain.vectorstores.faiss import FAISS | |
| from langchain.embeddings import OpenAIEmbeddings | |
| from langchain_community.document_loaders.mongodb import MongodbLoader | |
| from langchain.document_loaders import UnstructuredFileLoader | |
| import pymongo | |
| print("Loading data...") | |
| loader = UnstructuredFileLoader("state_of_the_union.txt") | |
| raw_documents = loader.load() | |
| print("Splitting text...") | |
| text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100) | |
| documents = text_splitter.split_documents(raw_documents) | |
| print("Creating vectorstore...") | |
| embeddings = OpenAIEmbeddings() | |
| vectorstore = FAISS.from_documents(documents, embeddings) | |
| vectorstore.save_local("faiss_index") | |