#!/usr/bin/env python3 """ Script to ingest documents into the vector store, assigning scopes based on subdirectories. """ import sys import argparse from pathlib import Path import shutil import logging import os # Add the app directory to the path so we can import app modules sys.path.append(str(Path(__file__).parent.parent)) from app.services.document_processor import DocumentProcessor from app.services.vector_store import VectorStoreManager from app.core.config import settings def main(): """Main ingestion function""" # Parse command line arguments parser = argparse.ArgumentParser(description="Ingest documents into vector store with scopes") parser.add_argument("--directory", "-d", default=settings.documents_directory_raw, help="Directory containing documents to ingest") parser.add_argument("--reset", "-r", action="store_true", help="Reset vector store before ingestion") parser.add_argument("--force", "-y", action="store_true", help="Force reset without confirmation") parser.add_argument("--verbose", "-v", action="store_true", help="Enable verbose logging") args = parser.parse_args() # Configure logging log_level = logging.DEBUG if args.verbose else logging.INFO logging.basicConfig( level=log_level, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' ) # Suppress noisy logs from third-party libraries logging.getLogger("chromadb.telemetry.product.posthog").setLevel(logging.CRITICAL) logger = logging.getLogger(__name__) print("=" * 60) print(" Document Ingestion Script (Multi-Scope)") print("=" * 60) project_root = Path(__file__).resolve().parent.parent # CRÍTICO: Muda o diretório de trabalho para a raiz do projeto # Isso garante que o VectorStoreManager crie o banco em 'project_root/data/chroma_db' e não dentro de 'scripts/' os.chdir(project_root) # Handle vector store reset if requested if args.reset: print("\n⚠️ Resetting vector store...") confirm = 'y' if args.force else input("This will delete all documents and clear the DB folder. Continue? (y/N): ") if confirm.lower() == 'y': # Tenta obter o diretório do banco das configurações ou usa o padrão 'data/chroma_db' db_path_setting = getattr(settings, 'persist_directory', 'data/chroma_db') db_path = Path(db_path_setting) if not db_path.is_absolute(): db_path = project_root / db_path if db_path.exists(): try: shutil.rmtree(db_path) print(f"✅ Diretório do banco removido: {db_path}") except Exception as e: print(f"❌ Erro ao remover diretório: {e}") else: print(f"⚠️ Diretório não encontrado: {db_path}") else: print("❌ Reset cancelled.") return # Initialize services vector_store = VectorStoreManager() processor = DocumentProcessor() # Resolve the documents directory path docs_arg = Path(args.directory).expanduser() # If path is relative, make it relative to project root docs_dir = (project_root / docs_arg if not docs_arg.is_absolute() else docs_arg).resolve(strict=False) if not docs_dir.exists(): print(f"\n❌ Documents directory not found: {docs_dir}") return print(f"\n📂 Processing directory: {docs_dir}") print(" (Subdirectories will be used as 'scopes')") # Prepare processed directory early processed_dir_setting = Path(settings.documents_directory_processed) processed_dir = (project_root / processed_dir_setting if not processed_dir_setting.is_absolute() else processed_dir_setting).resolve(strict=False) processed_dir.mkdir(parents=True, exist_ok=True) print(f" (Processed files will be moved to: {processed_dir})") # Check for potential duplicates if not resetting if not args.reset: current_stats = vector_store.get_collection_stats() if current_stats.get('total_documents', 0) > 0: print("\n⚠️ AVISO: O banco vetorial não está vazio. A ingestão sem --reset pode criar duplicatas e confundir as respostas.") total_chunks_added = 0 files_processed_count = 0 # Iterate over files and process one by one for root, dirs, files in os.walk(docs_dir): for file in files: file_path = Path(root) / file # Skip hidden files if file.startswith('.'): continue try: # Process single file chunks = processor.process_file(file_path) if chunks: # Calcula o escopo baseado no subdiretório (ex: 'sistemas' em 'data/raw/sistemas/doc.pdf') relative_path = file_path.relative_to(docs_dir) if len(relative_path.parts) > 1: scope = relative_path.parts[0] else: scope = 'default' # Força a atualização do metadado 'scope' em todos os chunks for chunk in chunks: chunk.metadata['scope'] = scope logger.info(f"Processing file: {file_path} (Scope: {scope})") # Add to vector store immediately vector_store.add_documents(chunks) print(f"{len(chunks)} chunks adicionado") total_chunks_added += len(chunks) files_processed_count += 1 # Move file to processed directory relative_path = file_path.relative_to(docs_dir) dest_path = processed_dir / relative_path dest_path.parent.mkdir(parents=True, exist_ok=True) if dest_path.exists(): dest_path.unlink() shutil.move(str(file_path), str(dest_path)) else: logger.warning(f"No chunks generated for file: {file_path}") except Exception as e: logger.error(f"Error processing file {file_path}: {e}") if files_processed_count == 0: print(f"\n⚠️ No documents found or processed in {docs_dir}") else: print(f"\n✅ Ingestion complete! Added {total_chunks_added} chunks from {files_processed_count} files.") # Display statistics stats = vector_store.get_collection_stats() print(f"\n📊 Collection Statistics:") print(f" Total chunks: {stats.get('total_documents', 0)}") print(f" Collection name: {stats.get('collection_name', 'N/A')}") print("\n🎉 Done!") if __name__ == "__main__": main()