Spaces:
Sleeping
Sleeping
| """ | |
| Ingest PDFs/TXT files from ./train_data into the persistent ChromaDB store. | |
| Usage: | |
| AZURE_API_KEY=... .venv/bin/python ingest_train_data.py | |
| The script reuses the optimized chunking, batching, and metadata logic from | |
| app.py so the local ingestion matches the API retrieval behavior. | |
| """ | |
| import logging | |
| import sys | |
| from app import TRAIN_DOCS_DIR, collection, ingest_train_documents | |
| logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") | |
| logger = logging.getLogger(__name__) | |
| def main() -> int: | |
| if not TRAIN_DOCS_DIR.exists(): | |
| logger.error("Missing %s. Put the corpus PDFs/TXT files there first.", TRAIN_DOCS_DIR) | |
| return 1 | |
| before = collection.count() | |
| if before > 0: | |
| logger.info("Vector store already contains %s chunks; skipping ingestion.", before) | |
| return 0 | |
| ingest_train_documents() | |
| after = collection.count() | |
| logger.info("Ingestion finished. Chunks in vector store: %s", after) | |
| return 0 if after > 0 else 1 | |
| if __name__ == "__main__": | |
| sys.exit(main()) | |