Download scripts\ingest_documents.py from aionaion/chatdoc: direct link, hf CLI and curl.
- Browser
- Download file 7.11 kB
-
https://huggingface.co/spaces/aionaion/chatdoc/resolve/main/scripts%5Cingest_documents.py
- Command line
-
hf download 'hf://spaces/aionaion/chatdoc/scripts\ingest_documents.py'
-
curl -L -o 'scripts\ingest_documents.py' https://huggingface.co/spaces/aionaion/chatdoc/resolve/main/scripts%5Cingest_documents.py
7.11 kB
| #!/usr/bin/env python3 | |
| """ | |
| Script to ingest documents into the vector store, assigning scopes based on subdirectories. | |
| """ | |
| import sys | |
| import argparse | |
| from pathlib import Path | |
| import shutil | |
| import logging | |
| import os | |
| # Add the app directory to the path so we can import app modules | |
| sys.path.append(str(Path(__file__).parent.parent)) | |
| from app.services.document_processor import DocumentProcessor | |
| from app.services.vector_store import VectorStoreManager | |
| from app.core.config import settings | |
| def main(): | |
| """Main ingestion function""" | |
| # Parse command line arguments | |
| parser = argparse.ArgumentParser(description="Ingest documents into vector store with scopes") | |
| parser.add_argument("--directory", "-d", | |
| default=settings.documents_directory_raw, | |
| help="Directory containing documents to ingest") | |
| parser.add_argument("--reset", "-r", | |
| action="store_true", | |
| help="Reset vector store before ingestion") | |
| parser.add_argument("--force", "-y", | |
| action="store_true", | |
| help="Force reset without confirmation") | |
| parser.add_argument("--verbose", "-v", | |
| action="store_true", | |
| help="Enable verbose logging") | |
| args = parser.parse_args() | |
| # Configure logging | |
| log_level = logging.DEBUG if args.verbose else logging.INFO | |
| logging.basicConfig( | |
| level=log_level, | |
| format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' | |
| ) | |
| # Suppress noisy logs from third-party libraries | |
| logging.getLogger("chromadb.telemetry.product.posthog").setLevel(logging.CRITICAL) | |
| logger = logging.getLogger(__name__) | |
| print("=" * 60) | |
| print(" Document Ingestion Script (Multi-Scope)") | |
| print("=" * 60) | |
| project_root = Path(__file__).resolve().parent.parent | |
| # CRÍTICO: Muda o diretório de trabalho para a raiz do projeto | |
| # Isso garante que o VectorStoreManager crie o banco em 'project_root/data/chroma_db' e não dentro de 'scripts/' | |
| os.chdir(project_root) | |
| # Handle vector store reset if requested | |
| if args.reset: | |
| print("\n⚠️ Resetting vector store...") | |
| confirm = 'y' if args.force else input("This will delete all documents and clear the DB folder. Continue? (y/N): ") | |
| if confirm.lower() == 'y': | |
| # Tenta obter o diretório do banco das configurações ou usa o padrão 'data/chroma_db' | |
| db_path_setting = getattr(settings, 'persist_directory', 'data/chroma_db') | |
| db_path = Path(db_path_setting) | |
| if not db_path.is_absolute(): | |
| db_path = project_root / db_path | |
| if db_path.exists(): | |
| try: | |
| shutil.rmtree(db_path) | |
| print(f"✅ Diretório do banco removido: {db_path}") | |
| except Exception as e: | |
| print(f"❌ Erro ao remover diretório: {e}") | |
| else: | |
| print(f"⚠️ Diretório não encontrado: {db_path}") | |
| else: | |
| print("❌ Reset cancelled.") | |
| return | |
| # Initialize services | |
| vector_store = VectorStoreManager() | |
| processor = DocumentProcessor() | |
| # Resolve the documents directory path | |
| docs_arg = Path(args.directory).expanduser() | |
| # If path is relative, make it relative to project root | |
| docs_dir = (project_root / docs_arg if not docs_arg.is_absolute() else docs_arg).resolve(strict=False) | |
| if not docs_dir.exists(): | |
| print(f"\n❌ Documents directory not found: {docs_dir}") | |
| return | |
| print(f"\n📂 Processing directory: {docs_dir}") | |
| print(" (Subdirectories will be used as 'scopes')") | |
| # Prepare processed directory early | |
| processed_dir_setting = Path(settings.documents_directory_processed) | |
| processed_dir = (project_root / processed_dir_setting if not processed_dir_setting.is_absolute() else processed_dir_setting).resolve(strict=False) | |
| processed_dir.mkdir(parents=True, exist_ok=True) | |
| print(f" (Processed files will be moved to: {processed_dir})") | |
| # Check for potential duplicates if not resetting | |
| if not args.reset: | |
| current_stats = vector_store.get_collection_stats() | |
| if current_stats.get('total_documents', 0) > 0: | |
| print("\n⚠️ AVISO: O banco vetorial não está vazio. A ingestão sem --reset pode criar duplicatas e confundir as respostas.") | |
| total_chunks_added = 0 | |
| files_processed_count = 0 | |
| # Iterate over files and process one by one | |
| for root, dirs, files in os.walk(docs_dir): | |
| for file in files: | |
| file_path = Path(root) / file | |
| # Skip hidden files | |
| if file.startswith('.'): | |
| continue | |
| try: | |
| # Process single file | |
| chunks = processor.process_file(file_path) | |
| if chunks: | |
| # Calcula o escopo baseado no subdiretório (ex: 'sistemas' em 'data/raw/sistemas/doc.pdf') | |
| relative_path = file_path.relative_to(docs_dir) | |
| if len(relative_path.parts) > 1: | |
| scope = relative_path.parts[0] | |
| else: | |
| scope = 'default' | |
| # Força a atualização do metadado 'scope' em todos os chunks | |
| for chunk in chunks: | |
| chunk.metadata['scope'] = scope | |
| logger.info(f"Processing file: {file_path} (Scope: {scope})") | |
| # Add to vector store immediately | |
| vector_store.add_documents(chunks) | |
| print(f"{len(chunks)} chunks adicionado") | |
| total_chunks_added += len(chunks) | |
| files_processed_count += 1 | |
| # Move file to processed directory | |
| relative_path = file_path.relative_to(docs_dir) | |
| dest_path = processed_dir / relative_path | |
| dest_path.parent.mkdir(parents=True, exist_ok=True) | |
| if dest_path.exists(): | |
| dest_path.unlink() | |
| shutil.move(str(file_path), str(dest_path)) | |
| else: | |
| logger.warning(f"No chunks generated for file: {file_path}") | |
| except Exception as e: | |
| logger.error(f"Error processing file {file_path}: {e}") | |
| if files_processed_count == 0: | |
| print(f"\n⚠️ No documents found or processed in {docs_dir}") | |
| else: | |
| print(f"\n✅ Ingestion complete! Added {total_chunks_added} chunks from {files_processed_count} files.") | |
| # Display statistics | |
| stats = vector_store.get_collection_stats() | |
| print(f"\n📊 Collection Statistics:") | |
| print(f" Total chunks: {stats.get('total_documents', 0)}") | |
| print(f" Collection name: {stats.get('collection_name', 'N/A')}") | |
| print("\n🎉 Done!") | |
| if __name__ == "__main__": | |
| main() | |