Spaces:
Sleeping
Sleeping
| import os | |
| from langchain_community.document_loaders import DirectoryLoader | |
| from langchain_community.vectorstores import FAISS | |
| from langchain_huggingface import HuggingFaceEmbeddings | |
| from rag_pipeline import process_single_file # Re-use our processing logic | |
| from config import settings | |
| def main(): | |
| print("Starting creation of the fallback FAISS index...") | |
| # This assumes your general documents are in a folder named 'fallback_documents' | |
| # You can change this path as needed. | |
| source_directory = "fallback_documents" | |
| all_docs = [] | |
| for filename in os.listdir(source_directory): | |
| file_path = os.path.join(source_directory, filename) | |
| if os.path.isfile(file_path): | |
| with open(file_path, "rb") as f: | |
| content = f.read() | |
| docs = process_single_file(content, filename) | |
| all_docs.extend(docs) | |
| if not all_docs: | |
| print("No documents found to index. Exiting.") | |
| return | |
| print(f"Generating embeddings for {len(all_docs)} chunks in the fallback documents...") | |
| embeddings = HuggingFaceEmbeddings(model_name=settings.EMBEDDING_MODEL_NAME) | |
| # Create the FAISS index from the documents | |
| vector_store = FAISS.from_documents(all_docs, embeddings) | |
| # Save the index to the path specified in your config | |
| # Ensure this path is inside your persistent storage volume (e.g., /data) | |
| fallback_path = settings.FALLBACK_INDEX_PATH | |
| os.makedirs(fallback_path, exist_ok=True) | |
| vector_store.save_local(fallback_path) | |
| print(f"Fallback index created and saved successfully at: {fallback_path}") | |
| if __name__ == "__main__": | |
| main() |