File size: 7,110 Bytes
78de529 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 | #!/usr/bin/env python3
"""
Script to ingest documents into the vector store, assigning scopes based on subdirectories.
"""
import sys
import argparse
from pathlib import Path
import shutil
import logging
import os
# Add the app directory to the path so we can import app modules
sys.path.append(str(Path(__file__).parent.parent))
from app.services.document_processor import DocumentProcessor
from app.services.vector_store import VectorStoreManager
from app.core.config import settings
def main():
"""Main ingestion function"""
# Parse command line arguments
parser = argparse.ArgumentParser(description="Ingest documents into vector store with scopes")
parser.add_argument("--directory", "-d",
default=settings.documents_directory_raw,
help="Directory containing documents to ingest")
parser.add_argument("--reset", "-r",
action="store_true",
help="Reset vector store before ingestion")
parser.add_argument("--force", "-y",
action="store_true",
help="Force reset without confirmation")
parser.add_argument("--verbose", "-v",
action="store_true",
help="Enable verbose logging")
args = parser.parse_args()
# Configure logging
log_level = logging.DEBUG if args.verbose else logging.INFO
logging.basicConfig(
level=log_level,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
# Suppress noisy logs from third-party libraries
logging.getLogger("chromadb.telemetry.product.posthog").setLevel(logging.CRITICAL)
logger = logging.getLogger(__name__)
print("=" * 60)
print(" Document Ingestion Script (Multi-Scope)")
print("=" * 60)
project_root = Path(__file__).resolve().parent.parent
# CRÍTICO: Muda o diretório de trabalho para a raiz do projeto
# Isso garante que o VectorStoreManager crie o banco em 'project_root/data/chroma_db' e não dentro de 'scripts/'
os.chdir(project_root)
# Handle vector store reset if requested
if args.reset:
print("\n⚠️ Resetting vector store...")
confirm = 'y' if args.force else input("This will delete all documents and clear the DB folder. Continue? (y/N): ")
if confirm.lower() == 'y':
# Tenta obter o diretório do banco das configurações ou usa o padrão 'data/chroma_db'
db_path_setting = getattr(settings, 'persist_directory', 'data/chroma_db')
db_path = Path(db_path_setting)
if not db_path.is_absolute():
db_path = project_root / db_path
if db_path.exists():
try:
shutil.rmtree(db_path)
print(f"✅ Diretório do banco removido: {db_path}")
except Exception as e:
print(f"❌ Erro ao remover diretório: {e}")
else:
print(f"⚠️ Diretório não encontrado: {db_path}")
else:
print("❌ Reset cancelled.")
return
# Initialize services
vector_store = VectorStoreManager()
processor = DocumentProcessor()
# Resolve the documents directory path
docs_arg = Path(args.directory).expanduser()
# If path is relative, make it relative to project root
docs_dir = (project_root / docs_arg if not docs_arg.is_absolute() else docs_arg).resolve(strict=False)
if not docs_dir.exists():
print(f"\n❌ Documents directory not found: {docs_dir}")
return
print(f"\n📂 Processing directory: {docs_dir}")
print(" (Subdirectories will be used as 'scopes')")
# Prepare processed directory early
processed_dir_setting = Path(settings.documents_directory_processed)
processed_dir = (project_root / processed_dir_setting if not processed_dir_setting.is_absolute() else processed_dir_setting).resolve(strict=False)
processed_dir.mkdir(parents=True, exist_ok=True)
print(f" (Processed files will be moved to: {processed_dir})")
# Check for potential duplicates if not resetting
if not args.reset:
current_stats = vector_store.get_collection_stats()
if current_stats.get('total_documents', 0) > 0:
print("\n⚠️ AVISO: O banco vetorial não está vazio. A ingestão sem --reset pode criar duplicatas e confundir as respostas.")
total_chunks_added = 0
files_processed_count = 0
# Iterate over files and process one by one
for root, dirs, files in os.walk(docs_dir):
for file in files:
file_path = Path(root) / file
# Skip hidden files
if file.startswith('.'):
continue
try:
# Process single file
chunks = processor.process_file(file_path)
if chunks:
# Calcula o escopo baseado no subdiretório (ex: 'sistemas' em 'data/raw/sistemas/doc.pdf')
relative_path = file_path.relative_to(docs_dir)
if len(relative_path.parts) > 1:
scope = relative_path.parts[0]
else:
scope = 'default'
# Força a atualização do metadado 'scope' em todos os chunks
for chunk in chunks:
chunk.metadata['scope'] = scope
logger.info(f"Processing file: {file_path} (Scope: {scope})")
# Add to vector store immediately
vector_store.add_documents(chunks)
print(f"{len(chunks)} chunks adicionado")
total_chunks_added += len(chunks)
files_processed_count += 1
# Move file to processed directory
relative_path = file_path.relative_to(docs_dir)
dest_path = processed_dir / relative_path
dest_path.parent.mkdir(parents=True, exist_ok=True)
if dest_path.exists():
dest_path.unlink()
shutil.move(str(file_path), str(dest_path))
else:
logger.warning(f"No chunks generated for file: {file_path}")
except Exception as e:
logger.error(f"Error processing file {file_path}: {e}")
if files_processed_count == 0:
print(f"\n⚠️ No documents found or processed in {docs_dir}")
else:
print(f"\n✅ Ingestion complete! Added {total_chunks_added} chunks from {files_processed_count} files.")
# Display statistics
stats = vector_store.get_collection_stats()
print(f"\n📊 Collection Statistics:")
print(f" Total chunks: {stats.get('total_documents', 0)}")
print(f" Collection name: {stats.get('collection_name', 'N/A')}")
print("\n🎉 Done!")
if __name__ == "__main__":
main()
|