#!/usr/bin/env python3 """ CLI for running ingestion operations. Usage: python -m cli.ingest sync python -m cli.ingest scrape --source youtube --company "Anthropic" python -m cli.ingest process --company "Anthropic" --chunks --claims python -m cli.ingest status python -m cli.ingest validate Examples: # Run a full sync of all files in the configured Google Drive folder python -m cli.ingest sync # Scrape YouTube for a specific company python -m cli.ingest scrape --source youtube --company "Anthropic" # Scrape YouTube for all tracked companies (dry run) python -m cli.ingest scrape --source youtube --companies all --dry-run # Generate chunks and claims for a company's documents python -m cli.ingest process --company "Anthropic" --chunks --claims # Re-process a specific document (delete existing and regenerate) python -m cli.ingest process --document-id "uuid" --chunks --claims --force # Process documents missing claims python -m cli.ingest process --missing claims --claims --limit 50 # Validate configuration without running sync python -m cli.ingest validate # Check status of recent ingestion jobs python -m cli.ingest status """ import argparse import logging import sys from config.ingestion import load_ingestion_config from config.scraping import load_scraping_config from services.ingestion.orchestrator import IngestionConfig, IngestionOrchestrator from services.kg import KGClient, get_kg_client from services.sources.base import ContentSource from services.sources.gdrive import GoogleDriveSource from services.sources.local import LocalFileSource from services.storage.base import StorageBackend from services.storage.gcs import GCSStorageBackend from services.storage.noop import NoOpStorageBackend # Configure logging logging.basicConfig( level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s" ) logger = logging.getLogger(__name__) def validate_config(): """Validate configuration and return errors.""" config = load_ingestion_config() errors = config.validate() if errors: print("Configuration errors:") for error in errors: print(f" - {error}") return False print("Configuration is valid:") print(f" - Google Drive folder: {config.gdrive.root_folder_id}") print(f" - GCS bucket: {config.gcs.bucket_name}") print(f" - Company folder depth: {config.gdrive.company_folder_depth}") print(f" - Chunk size: {config.chunking.chunk_size}") print() print("Claim extraction settings:") print(f" - Enabled: {config.claim_extraction.enabled}") print(f" - LLM provider: {config.claim_extraction.llm_provider}") print(f" - Model: {config.claim_extraction.model}") return True def run_sync(args): """Run sync operation.""" # Load and validate config config = load_ingestion_config() errors = config.validate() if errors: print("Configuration errors:") for error in errors: print(f" - {error}") sys.exit(1) # Initialize Google Drive source print("Initializing Google Drive connection...") gdrive = GoogleDriveSource( credentials_path=config.gdrive.credentials_path, credentials_dict=config.gdrive.get_credentials_dict(), ) if not gdrive.authenticate(): print("ERROR: Failed to authenticate with Google Drive") sys.exit(1) print(" Google Drive authenticated successfully") # Initialize GCS storage print("Initializing GCS storage...") gcs = GCSStorageBackend( bucket_name=config.gcs.bucket_name, project_id=config.gcs.project_id, credentials_path=config.gcs.credentials_path, prefix=config.gcs.prefix, ) print(f" GCS bucket: gs://{config.gcs.bucket_name}/{config.gcs.prefix}") # Initialize KG client print("Connecting to Supabase...") kg = get_kg_client() print(" Supabase connected successfully") # Determine what processing to enable enable_chunking = not getattr(args, "no_chunks", False) enable_claims = config.claim_extraction.enabled and not getattr(args, "no_claims", False) # Show processing settings print() print("Processing settings:") print(f" Chunking: {'enabled' if enable_chunking else 'DISABLED (--no-chunks)'}") print(f" Claim extraction: {'enabled' if enable_claims else 'disabled'}") if enable_claims: print(f" Provider: {config.claim_extraction.llm_provider}") print(f" Model: {config.claim_extraction.model}") # Get company override if specified override_company = getattr(args, "company", None) if override_company: print(f" Company override: {override_company}") # Create orchestrator orchestrator = IngestionOrchestrator( source=gdrive, storage=gcs, kg_client=kg, config=IngestionConfig( root_folder_id=config.gdrive.root_folder_id, company_folder_depth=config.gdrive.company_folder_depth, chunk_size=config.chunking.chunk_size, chunk_overlap=config.chunking.chunk_overlap, batch_size=config.batch_size, max_file_size_mb=config.max_file_size_mb, enable_chunking=enable_chunking, enable_claim_extraction=enable_claims, claim_extraction_llm=config.claim_extraction.llm_provider, claim_extraction_model=config.claim_extraction.model, override_company_name=override_company, ), ) # Run sync print() print("=" * 60) print("Running FULL SYNC") print("=" * 60) stats = orchestrator.run_full_sync() # Print results _print_sync_stats(stats) # Exit with error if there were failures if stats.files_failed > 0: sys.exit(1) def run_sync_local(args): """Run sync operation for local filesystem.""" # Validate path local_path = args.path if not local_path: print("ERROR: --path is required") sys.exit(1) # Load config (for GCS and chunking settings) config = load_ingestion_config() # Initialize local file source print(f"Initializing local filesystem source: {local_path}") local_source = LocalFileSource(root_path=local_path) if not local_source.authenticate(): print(f"ERROR: Failed to access local path: {local_path}") sys.exit(1) print(" Local filesystem authenticated successfully") # Initialize storage backend use_noop_storage = getattr(args, "no_storage", False) storage: StorageBackend if use_noop_storage: print("Initializing no-op storage (files will NOT be persisted)...") storage = NoOpStorageBackend(prefix="noop://local/") print(" Storage: no-op (dry run)") else: print("Initializing GCS storage...") storage = GCSStorageBackend( bucket_name=config.gcs.bucket_name, project_id=config.gcs.project_id, credentials_path=config.gcs.credentials_path, prefix=config.gcs.prefix, ) print(f" GCS bucket: gs://{config.gcs.bucket_name}/{config.gcs.prefix}") # Initialize KG client print("Connecting to Supabase...") kg = get_kg_client() print(" Supabase connected successfully") # Determine what processing to enable enable_chunking = not getattr(args, "no_chunks", False) enable_claims = config.claim_extraction.enabled and not getattr(args, "no_claims", False) # Show processing settings print() print("Processing settings:") print(f" Chunking: {'enabled' if enable_chunking else 'DISABLED (--no-chunks)'}") print(f" Claim extraction: {'enabled' if enable_claims else 'disabled'}") if enable_claims: print(f" Provider: {config.claim_extraction.llm_provider}") print(f" Model: {config.claim_extraction.model}") # Get company folder depth (use same config as gdrive, or override via args) company_folder_depth = ( getattr(args, "company_depth", None) or config.gdrive.company_folder_depth ) # Get company override if specified override_company = getattr(args, "company", None) if override_company: print(f" Company override: {override_company}") # Create orchestrator orchestrator = IngestionOrchestrator( source=local_source, storage=storage, kg_client=kg, config=IngestionConfig( root_folder_id=str(local_source.root_path), company_folder_depth=company_folder_depth, chunk_size=config.chunking.chunk_size, chunk_overlap=config.chunking.chunk_overlap, batch_size=config.batch_size, max_file_size_mb=config.max_file_size_mb, enable_chunking=enable_chunking, enable_claim_extraction=enable_claims, claim_extraction_llm=config.claim_extraction.llm_provider, claim_extraction_model=config.claim_extraction.model, override_company_name=override_company, ), ) # Run sync print() print("=" * 60) print(f"Running LOCAL SYNC: {local_path}") print("=" * 60) stats = orchestrator.run_full_sync() # Print results _print_sync_stats(stats) # Print storage stats if using no-op storage if use_noop_storage and isinstance(storage, NoOpStorageBackend): storage.print_summary() # Exit with error if there were failures if stats.files_failed > 0: sys.exit(1) def _print_sync_stats(stats): """Print detailed sync statistics.""" print() print("=" * 70) print("SYNC COMPLETED") print("=" * 70) # Overall summary print() print("SUMMARY") print("-" * 70) print(f" Files found: {stats.files_found}") print(f" Files processed: {stats.files_processed}") print(f" Files skipped: {stats.files_skipped}") print(f" Files failed: {stats.files_failed}") print(f" Chunks created: {stats.chunks_created}") print(f" Claims extracted: {stats.claims_extracted}") # Group documents by company docs_by_company: dict[str, list] = {} for doc in stats.documents: company = doc.company_name or "(No Company)" if company not in docs_by_company: docs_by_company[company] = [] docs_by_company[company].append(doc) # Per-company breakdown with nested documents if stats.companies or docs_by_company: print() print("BY COMPANY") print("-" * 70) for company_name in sorted(docs_by_company.keys()): company_docs = docs_by_company[company_name] company_stat = stats.companies.get(company_name) # Company header with stats processed = [d for d in company_docs if d.status == "processed"] skipped = [d for d in company_docs if d.status == "skipped"] failed = [d for d in company_docs if d.status == "failed"] print(f"\n {company_name}") print( f" Files: {len(company_docs)} total, " f"{len(processed)} processed, " f"{len(skipped)} skipped, " f"{len(failed)} failed" ) if company_stat and (company_stat.chunks_created or company_stat.claims_extracted): print( f" Output: {company_stat.chunks_created} chunks, " f"{company_stat.claims_extracted} claims" ) # Processed documents if processed: print() print(" Processed:") for doc in processed: print(f" - {doc.name}") print(f" ID: {doc.document_id}") print(f" {doc.chunks_created} chunks, {doc.claims_extracted} claims") # Failed documents if failed: print() print(" Failed:") for doc in failed: print(f" - {doc.name}") if doc.document_id: print(f" ID: {doc.document_id}") print(f" Error: {doc.error}") # Skipped documents (grouped by reason) if skipped: print() print(" Skipped:") by_reason: dict[str, list] = {} for doc in skipped: reason = doc.skip_reason or "unknown" if reason not in by_reason: by_reason[reason] = [] by_reason[reason].append(doc) for reason, docs in sorted(by_reason.items()): if len(docs) <= 3: for doc in docs: print(f" - {doc.name} ({reason})") else: print(f" - {len(docs)} files ({reason})") if stats.errors: print() print("ERRORS") print("-" * 70) for error in stats.errors[:10]: print(f" - {error}") if len(stats.errors) > 10: print(f" ... and {len(stats.errors) - 10} more errors") def show_status(args): """Show status of recent ingestion jobs.""" print("Connecting to Supabase...") kg = get_kg_client() # Query recent jobs resp = ( kg.client.table("ingestion_jobs") .select("*") .order("created_at", desc=True) .limit(10) .execute() ) if not resp.data: print("\nNo ingestion jobs found.") return print("\nRecent Ingestion Jobs:") print("-" * 80) for job in resp.data: created = job.get("created_at", "N/A")[:19] if job.get("created_at") else "N/A" status = job.get("status", "N/A") job_type = job.get("job_type", "N/A") source = job.get("source_type", "N/A") status_icon = {"succeeded": "✓", "failed": "✗", "running": "⟳", "queued": "○"}.get( status, "?" ) print(f"{status_icon} [{status:10}] {job_type:20} ({source}) - {created}") # Show payload summary if available payload = job.get("payload", {}) if isinstance(payload, dict): if "files_processed" in payload: print( f" Processed: {payload.get('files_processed', 0)}, " f"Skipped: {payload.get('files_skipped', 0)}, " f"Failed: {payload.get('files_failed', 0)}" ) if job.get("error_message"): print(f" Error: {job['error_message'][:100]}") print("-" * 80) SCRAPER_REGISTRY = { "youtube": "YouTubeScraper", } def run_process(args): """Process existing documents to generate chunks and/or claims.""" # Load config config = load_ingestion_config() # Initialize KG client print("Connecting to Supabase...") kg = get_kg_client() print(" Supabase connected successfully") # Show settings print() print("Processing settings:") print(f" Chunking: {'enabled' if args.chunks else 'disabled'}") print(f" Claims: {'enabled' if args.claims else 'disabled'}") if args.claims: print(f" Provider: {config.claim_extraction.llm_provider}") print(f" Model: {config.claim_extraction.model}") if not args.chunks and not args.claims: print("\nERROR: Must specify --chunks and/or --claims") sys.exit(1) # Build document query documents = [] if args.document_id: # Process specific document doc = kg.get_document_by_id(args.document_id) if not doc: print(f"ERROR: Document not found: {args.document_id}") sys.exit(1) documents = [doc] elif args.company: # Process documents for a company company = kg.get_company_by_name(args.company) if not company: print(f"ERROR: Company not found: {args.company}") sys.exit(1) documents = kg.get_documents_for_company(company.id, limit=args.limit) elif args.missing: # Process documents missing chunks or claims documents = _get_documents_missing_processing(kg, args.missing, args.limit) else: print("ERROR: Must specify --document-id, --company, or --missing") sys.exit(1) if not documents: print("\nNo documents found to process") return print(f"\nDocuments to process: {len(documents)}") # Initialize processors from hashlib import sha256 from llama_index.core import Document as LlamaDoc from llama_index.core.node_parser import SentenceSplitter from llama_index.embeddings.openai import OpenAIEmbedding from models.internal import SourceType from services.claims import ClaimExtractor from services.llm import GeminiLLM, OpenAiLLM claim_extractor = None if args.claims: if config.claim_extraction.llm_provider == "openai": api_key = config.llm.openai_api_key if not api_key: print("ERROR: OPENAI_API_KEY required for claim extraction") sys.exit(1) llm = OpenAiLLM(api_key=api_key, model_name=config.claim_extraction.model) else: api_key = config.llm.gemini_api_key if not api_key: print("ERROR: GEMINI_API_KEY required for claim extraction") sys.exit(1) llm = GeminiLLM(api_key=api_key, model_name=config.claim_extraction.model) claim_extractor = ClaimExtractor(llm=llm) embed_model = None splitter = None if args.chunks: embed_model = OpenAIEmbedding(model="text-embedding-3-small") splitter = SentenceSplitter( chunk_size=config.chunking.chunk_size, chunk_overlap=config.chunking.chunk_overlap, ) # Create job for tracking job_id = kg.create_ingestion_job( job_type="process", source_type="existing_documents", payload={ "chunks_enabled": args.chunks, "claims_enabled": args.claims, "document_count": len(documents), }, ) # Process documents total_processed = 0 total_skipped = 0 total_failed = 0 total_chunks = 0 total_claims = 0 for doc in documents: print(f"\nProcessing: {doc.title or doc.id}") if not doc.parsed_text: print(" Skipped: no parsed_text") total_skipped += 1 continue try: # Generate chunks if args.chunks and splitter and embed_model: # Delete existing chunks if re-processing if args.force: kg.delete_chunks_for_document(doc.id) llama_doc = LlamaDoc(text=doc.parsed_text) nodes = splitter.get_nodes_from_documents([llama_doc]) if nodes: chunks_to_insert = [] for idx, node in enumerate(nodes): chunk_text = node.text chunk_hash = sha256(chunk_text.encode()).hexdigest() embedding = embed_model.get_text_embedding(chunk_text) chunks_to_insert.append( { "document_id": doc.id, "company_id": doc.company_id, "chunk_index": idx, "chunk_text": chunk_text, "chunk_hash": chunk_hash, "embedding": embedding, "chunk_kind": "document_section", } ) kg.create_chunks_batch(chunks_to_insert) total_chunks += len(chunks_to_insert) print(f" Created {len(chunks_to_insert)} chunks") # Extract claims if args.claims and claim_extractor: # Delete existing claims if re-processing if args.force: kg.delete_claims_for_document(doc.id) # Infer source type from document source_type = ( SourceType(doc.source_type) if doc.source_type else SourceType.UNDEFINED ) claims = claim_extractor.extract( text=doc.parsed_text, source_type=source_type, document_id=doc.id, ) if claims: kg_claims = claim_extractor.to_kg_format( claims=claims, document_id=doc.id, company_id=doc.company_id, ) count = kg.create_claims_batch(kg_claims) total_claims += count print(f" Extracted {count} claims") total_processed += 1 except Exception as e: total_failed += 1 print(f" Error: {e}") logger.exception(f"Failed to process document {doc.id}") # Update job kg.update_ingestion_job( job_id, status="succeeded" if total_failed == 0 else "completed_with_errors", payload={ "documents_processed": total_processed, "documents_skipped": total_skipped, "documents_failed": total_failed, "chunks_created": total_chunks, "claims_extracted": total_claims, }, ) # Summary print() print("=" * 60) print("PROCESSING COMPLETED") print("=" * 60) print(f" Documents processed: {total_processed}") print(f" Documents skipped: {total_skipped}") print(f" Documents failed: {total_failed}") print(f" Chunks created: {total_chunks}") print(f" Claims extracted: {total_claims}") def _get_documents_missing_processing(kg: KGClient, missing_type: str, limit: int) -> list: """Get documents missing chunks or claims.""" if missing_type == "chunks": # Documents with parsed_text but no chunks resp = kg.client.rpc("get_documents_without_chunks", {"p_limit": limit}).execute() elif missing_type == "claims": # Documents with parsed_text but no claims resp = kg.client.rpc("get_documents_without_claims", {"p_limit": limit}).execute() else: # Try raw query as fallback resp = ( kg.client.table("documents") .select("*") .not_.is_("parsed_text", "null") .limit(limit) .execute() ) return [kg._row_to_document(row) for row in resp.data] def _build_scraper(source: str, scraping_config, kg) -> ContentSource: """Build scraper instance for the given source.""" if source == "youtube": if not scraping_config.youtube_api_key: print("ERROR: YOUTUBE_API_KEY environment variable required") sys.exit(1) from services.scrapers.youtube import YouTubeScraper return YouTubeScraper(scraping_config.youtube_api_key, kg, scraping_config.youtube) raise ValueError(f"Unknown source: {source}") def run_scrape(args): """Run scrape operation for public sources using the full ingestion pipeline.""" if not args.company and not args.companies: print("ERROR: Must specify --company or --companies all") sys.exit(1) # Load configs config = load_ingestion_config() scraping_config = load_scraping_config() # Initialize KG client print("Connecting to Supabase...") kg = get_kg_client() print(" Supabase connected successfully") # Build scraper print(f"Initializing {args.source} scraper...") scraper = _build_scraper(args.source, scraping_config, kg) if not scraper.authenticate(): print(f"ERROR: Failed to authenticate {args.source}") sys.exit(1) print(f" {args.source} authenticated successfully") # Initialize GCS storage for pipeline print("Initializing GCS storage...") gcs = GCSStorageBackend( bucket_name=config.gcs.bucket_name, project_id=config.gcs.project_id, credentials_path=config.gcs.credentials_path, prefix=config.gcs.prefix, ) print(f" GCS bucket: gs://{config.gcs.bucket_name}/{config.gcs.prefix}") # Show claim extraction settings print() print("Claim extraction:") print(f" Enabled: {config.claim_extraction.enabled}") if config.claim_extraction.enabled: print(f" Provider: {config.claim_extraction.llm_provider}") print(f" Model: {config.claim_extraction.model}") # Get target companies if args.company: company_obj = kg.get_company_by_name(args.company) if not company_obj: print(f"ERROR: Company not found: {args.company}") sys.exit(1) target_companies = [company_obj] else: # Get all companies from KG resp = kg.client.table("companies").select("*").execute() from services.kg import KGCompany target_companies = [ KGCompany( id=row["id"], name=row["name"], normalized_name=row["normalized_name"], domain=row.get("domain"), aliases=row.get("aliases"), external_ids=row.get("external_ids"), metadata=row.get("metadata"), ) for row in resp.data ] if not target_companies: print("No companies found to scrape") return print(f"\nTarget companies: {len(target_companies)}") # Track detailed stats total_found = 0 total_processed = 0 total_skipped = 0 total_chunks = 0 total_claims = 0 total_failed = 0 # Per-company and per-document tracking company_stats: dict[str, dict] = {} processed_docs: list[dict] = [] failed_docs: list[dict] = [] skipped_docs: list[dict] = [] for company in target_companies: print(f"\nScraping {args.source} for: {company.name}") items = list(scraper.list_folder(company.id)) company_found = len(items) total_found += company_found print(f" Found {company_found} items") # Initialize company stats company_stats[company.name] = { "found": company_found, "processed": 0, "skipped": 0, "failed": 0, "chunks": 0, "claims": 0, } if args.dry_run: for item in items[: args.limit]: print(f" Would scrape: {item.name}") if hasattr(item, "metadata") and item.metadata.get("url"): print(f" URL: {item.metadata['url']}") else: # Create orchestrator for this company's batch orchestrator = IngestionOrchestrator( source=scraper, storage=gcs, kg_client=kg, config=IngestionConfig( root_folder_id=company.id, company_folder_depth=1, chunk_size=config.chunking.chunk_size, chunk_overlap=config.chunking.chunk_overlap, batch_size=config.batch_size, max_file_size_mb=config.max_file_size_mb, enable_claim_extraction=config.claim_extraction.enabled, claim_extraction_llm=config.claim_extraction.llm_provider, claim_extraction_model=config.claim_extraction.model, ), ) # Create ingestion job for tracking job_id = kg.create_ingestion_job( job_type="scrape", source_type=scraper.source_type, payload={"company_id": company.id, "company_name": company.name}, ) for item in items[: args.limit]: print(f" Processing: {item.name}") try: result = orchestrator._process_file(item, job_id) if result.skipped: reason = result.skip_reason or "unknown" print(f" Skipped: {reason}") total_skipped += 1 company_stats[company.name]["skipped"] += 1 skipped_docs.append( { "name": item.name, "company": company.name, "reason": reason, } ) elif result.success: total_processed += 1 total_chunks += result.chunk_count total_claims += result.claim_count company_stats[company.name]["processed"] += 1 company_stats[company.name]["chunks"] += result.chunk_count company_stats[company.name]["claims"] += result.claim_count print(f" Done: {result.chunk_count} chunks, {result.claim_count} claims") processed_docs.append( { "name": item.name, "company": company.name, "document_id": result.document_id, "chunks": result.chunk_count, "claims": result.claim_count, } ) else: total_failed += 1 company_stats[company.name]["failed"] += 1 print(f" Failed: {result.error}") failed_docs.append( { "name": item.name, "company": company.name, "error": result.error, } ) except Exception as e: total_failed += 1 company_stats[company.name]["failed"] += 1 print(f" Error: {e}") failed_docs.append( { "name": item.name, "company": company.name, "error": str(e), } ) # Update job status kg.update_ingestion_job( job_id, status="succeeded" if company_stats[company.name]["failed"] == 0 else "completed_with_errors", payload={ "company_id": company.id, "company_name": company.name, "files_found": company_found, "files_processed": company_stats[company.name]["processed"], "files_skipped": company_stats[company.name]["skipped"], "files_failed": company_stats[company.name]["failed"], "chunks_created": company_stats[company.name]["chunks"], "claims_extracted": company_stats[company.name]["claims"], }, ) # Print detailed summary _print_scrape_stats( args, total_found, total_processed, total_skipped, total_failed, total_chunks, total_claims, company_stats, processed_docs, failed_docs, skipped_docs, ) def _print_scrape_stats( args, total_found, total_processed, total_skipped, total_failed, total_chunks, total_claims, company_stats, processed_docs, failed_docs, skipped_docs, ): """Print detailed scrape statistics.""" print() print("=" * 70) print("SCRAPE COMPLETED") print("=" * 70) # Overall summary print() print("SUMMARY") print("-" * 70) print(f" Items found: {total_found}") if not args.dry_run: print(f" Items processed: {total_processed}") print(f" Items skipped: {total_skipped}") print(f" Items failed: {total_failed}") print(f" Chunks created: {total_chunks}") print(f" Claims extracted: {total_claims}") if args.dry_run: return # Per-company breakdown with nested documents if company_stats: print() print("BY COMPANY") print("-" * 70) for company_name, stats in sorted(company_stats.items()): # Company header print(f"\n {company_name}") print( f" Items: {stats['found']} found, " f"{stats['processed']} processed, " f"{stats['skipped']} skipped, " f"{stats['failed']} failed" ) if stats["chunks"] or stats["claims"]: print(f" Output: {stats['chunks']} chunks, {stats['claims']} claims") # Processed documents for this company company_processed = [d for d in processed_docs if d["company"] == company_name] if company_processed: print() print(" Processed:") for doc in company_processed: print(f" - {doc['name']}") print(f" ID: {doc['document_id']}") print(f" {doc['chunks']} chunks, {doc['claims']} claims") # Failed documents for this company company_failed = [d for d in failed_docs if d["company"] == company_name] if company_failed: print() print(" Failed:") for doc in company_failed: print(f" - {doc['name']}") print(f" Error: {doc['error']}") # Skipped documents for this company company_skipped = [d for d in skipped_docs if d["company"] == company_name] if company_skipped: print() print(" Skipped:") # Group by reason by_reason: dict[str, list] = {} for doc in company_skipped: reason = doc["reason"] if reason not in by_reason: by_reason[reason] = [] by_reason[reason].append(doc) for reason, docs in sorted(by_reason.items()): if len(docs) <= 3: for doc in docs: print(f" - {doc['name']} ({reason})") else: print(f" - {len(docs)} items ({reason})") def main(): parser = argparse.ArgumentParser( description="Knowledge Graph Ingestion CLI", formatter_class=argparse.RawDescriptionHelpFormatter, epilog=__doc__, ) subparsers = parser.add_subparsers(dest="command") # Sync command (Google Drive) sync_parser = subparsers.add_parser("sync", help="Run full sync of all files from Google Drive") sync_parser.add_argument( "--no-chunks", action="store_true", help="Skip chunking and embedding generation" ) sync_parser.add_argument("--no-claims", action="store_true", help="Skip claim extraction") sync_parser.add_argument( "--company", type=str, help="Override company for all documents (creates company if doesn't exist)", ) # Sync-local command (Local filesystem) sync_local_parser = subparsers.add_parser( "sync-local", help="Run full sync of files from local filesystem" ) sync_local_parser.add_argument( "--path", type=str, required=True, help="Path to local directory to ingest" ) sync_local_parser.add_argument( "--no-chunks", action="store_true", help="Skip chunking and embedding generation" ) sync_local_parser.add_argument("--no-claims", action="store_true", help="Skip claim extraction") sync_local_parser.add_argument( "--company-depth", type=int, help="Folder depth where company names appear (default: uses GDRIVE_COMPANY_FOLDER_DEPTH)", ) sync_local_parser.add_argument( "--no-storage", action="store_true", help="Skip uploading files to GCS (useful for testing)", ) sync_local_parser.add_argument( "--company", type=str, help="Override company for all documents (creates company if doesn't exist)", ) # Scrape command scrape_parser = subparsers.add_parser("scrape", help="Scrape public sources for content") scrape_parser.add_argument( "--source", type=str, choices=list(SCRAPER_REGISTRY.keys()), required=True, help="Source to scrape", ) scrape_parser.add_argument("--company", type=str, help="Specific company name") scrape_parser.add_argument( "--companies", type=str, choices=["all"], help="All tracked companies" ) scrape_parser.add_argument("--limit", type=int, default=20, help="Max items per company") scrape_parser.add_argument("--dry-run", action="store_true", help="Show what would be scraped") # Process command process_parser = subparsers.add_parser( "process", help="Generate chunks and/or claims from existing documents" ) process_parser.add_argument("--document-id", type=str, help="Process a specific document by ID") process_parser.add_argument("--company", type=str, help="Process all documents for a company") process_parser.add_argument( "--missing", type=str, choices=["chunks", "claims"], help="Process documents missing chunks or claims", ) process_parser.add_argument( "--chunks", action="store_true", help="Generate chunks with embeddings" ) process_parser.add_argument( "--claims", action="store_true", help="Extract claims from documents" ) process_parser.add_argument( "--force", action="store_true", help="Delete existing chunks/claims before re-processing", ) process_parser.add_argument("--limit", type=int, default=100, help="Max documents to process") # Validate command validate_parser = subparsers.add_parser("validate", help="Validate configuration") # Status command status_parser = subparsers.add_parser("status", help="Check status of recent ingestion jobs") args = parser.parse_args() if args.command == "sync": run_sync(args) elif args.command == "sync-local": run_sync_local(args) elif args.command == "scrape": run_scrape(args) elif args.command == "process": run_process(args) elif args.command == "validate": if not validate_config(): sys.exit(1) elif args.command == "status": show_status(args) else: parser.print_help() sys.exit(1) if __name__ == "__main__": main()