Spaces:
Sleeping
Sleeping
File size: 1,066 Bytes
5f5e6b2 6a95ad6 5f5e6b2 6a95ad6 5f5e6b2 6a95ad6 5f5e6b2 6a95ad6 5f5e6b2 6a95ad6 5f5e6b2 6a95ad6 5f5e6b2 6a95ad6 5f5e6b2 6a95ad6 5f5e6b2 6a95ad6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 | """
Ingest PDFs/TXT files from ./train_data into the persistent ChromaDB store.
Usage:
AZURE_API_KEY=... .venv/bin/python ingest_train_data.py
The script reuses the optimized chunking, batching, and metadata logic from
app.py so the local ingestion matches the API retrieval behavior.
"""
import logging
import sys
from app import TRAIN_DOCS_DIR, collection, ingest_train_documents
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
logger = logging.getLogger(__name__)
def main() -> int:
if not TRAIN_DOCS_DIR.exists():
logger.error("Missing %s. Put the corpus PDFs/TXT files there first.", TRAIN_DOCS_DIR)
return 1
before = collection.count()
if before > 0:
logger.info("Vector store already contains %s chunks; skipping ingestion.", before)
return 0
ingest_train_documents()
after = collection.count()
logger.info("Ingestion finished. Chunks in vector store: %s", after)
return 0 if after > 0 else 1
if __name__ == "__main__":
sys.exit(main())
|