Download cli/ingest.py from Altisvc/day1demo: direct link, hf CLI and curl.
- Browser
- Download file 39.4 kB
-
https://huggingface.co/spaces/Altisvc/day1demo/resolve/main/cli/ingest.py
- Command line
-
hf download hf://spaces/Altisvc/day1demo/cli/ingest.py
-
curl -L -o ingest.py https://huggingface.co/spaces/Altisvc/day1demo/resolve/main/cli/ingest.py
39.4 kB
#!/usr/bin/env python3
"""
CLI for running ingestion operations.
Usage:
python -m cli.ingest sync
python -m cli.ingest scrape --source youtube --company "Anthropic"
python -m cli.ingest process --company "Anthropic" --chunks --claims
python -m cli.ingest status
python -m cli.ingest validate
Examples:
# Run a full sync of all files in the configured Google Drive folder
python -m cli.ingest sync
# Scrape YouTube for a specific company
python -m cli.ingest scrape --source youtube --company "Anthropic"
# Scrape YouTube for all tracked companies (dry run)
python -m cli.ingest scrape --source youtube --companies all --dry-run
# Generate chunks and claims for a company's documents
python -m cli.ingest process --company "Anthropic" --chunks --claims
# Re-process a specific document (delete existing and regenerate)
python -m cli.ingest process --document-id "uuid" --chunks --claims --force
# Process documents missing claims
python -m cli.ingest process --missing claims --claims --limit 50
# Validate configuration without running sync
python -m cli.ingest validate
# Check status of recent ingestion jobs
python -m cli.ingest status
"""
import argparse
import logging
import sys
from config.ingestion import load_ingestion_config
from config.scraping import load_scraping_config
from services.ingestion.orchestrator import IngestionConfig, IngestionOrchestrator
from services.kg import KGClient, get_kg_client
from services.sources.base import ContentSource
from services.sources.gdrive import GoogleDriveSource
from services.sources.local import LocalFileSource
from services.storage.base import StorageBackend
from services.storage.gcs import GCSStorageBackend
from services.storage.noop import NoOpStorageBackend
# Configure logging
logging.basicConfig(
level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
)
logger = logging.getLogger(__name__)
def validate_config():
"""Validate configuration and return errors."""
config = load_ingestion_config()
errors = config.validate()
if errors:
print("Configuration errors:")
for error in errors:
print(f" - {error}")
return False
print("Configuration is valid:")
print(f" - Google Drive folder: {config.gdrive.root_folder_id}")
print(f" - GCS bucket: {config.gcs.bucket_name}")
print(f" - Company folder depth: {config.gdrive.company_folder_depth}")
print(f" - Chunk size: {config.chunking.chunk_size}")
print()
print("Claim extraction settings:")
print(f" - Enabled: {config.claim_extraction.enabled}")
print(f" - LLM provider: {config.claim_extraction.llm_provider}")
print(f" - Model: {config.claim_extraction.model}")
return True
def run_sync(args):
"""Run sync operation."""
# Load and validate config
config = load_ingestion_config()
errors = config.validate()
if errors:
print("Configuration errors:")
for error in errors:
print(f" - {error}")
sys.exit(1)
# Initialize Google Drive source
print("Initializing Google Drive connection...")
gdrive = GoogleDriveSource(
credentials_path=config.gdrive.credentials_path,
credentials_dict=config.gdrive.get_credentials_dict(),
)
if not gdrive.authenticate():
print("ERROR: Failed to authenticate with Google Drive")
sys.exit(1)
print(" Google Drive authenticated successfully")
# Initialize GCS storage
print("Initializing GCS storage...")
gcs = GCSStorageBackend(
bucket_name=config.gcs.bucket_name,
project_id=config.gcs.project_id,
credentials_path=config.gcs.credentials_path,
prefix=config.gcs.prefix,
)
print(f" GCS bucket: gs://{config.gcs.bucket_name}/{config.gcs.prefix}")
# Initialize KG client
print("Connecting to Supabase...")
kg = get_kg_client()
print(" Supabase connected successfully")
# Determine what processing to enable
enable_chunking = not getattr(args, "no_chunks", False)
enable_claims = config.claim_extraction.enabled and not getattr(args, "no_claims", False)
# Show processing settings
print()
print("Processing settings:")
print(f" Chunking: {'enabled' if enable_chunking else 'DISABLED (--no-chunks)'}")
print(f" Claim extraction: {'enabled' if enable_claims else 'disabled'}")
if enable_claims:
print(f" Provider: {config.claim_extraction.llm_provider}")
print(f" Model: {config.claim_extraction.model}")
# Get company override if specified
override_company = getattr(args, "company", None)
if override_company:
print(f" Company override: {override_company}")
# Create orchestrator
orchestrator = IngestionOrchestrator(
source=gdrive,
storage=gcs,
kg_client=kg,
config=IngestionConfig(
root_folder_id=config.gdrive.root_folder_id,
company_folder_depth=config.gdrive.company_folder_depth,
chunk_size=config.chunking.chunk_size,
chunk_overlap=config.chunking.chunk_overlap,
batch_size=config.batch_size,
max_file_size_mb=config.max_file_size_mb,
enable_chunking=enable_chunking,
enable_claim_extraction=enable_claims,
claim_extraction_llm=config.claim_extraction.llm_provider,
claim_extraction_model=config.claim_extraction.model,
override_company_name=override_company,
),
)
# Run sync
print()
print("=" * 60)
print("Running FULL SYNC")
print("=" * 60)
stats = orchestrator.run_full_sync()
# Print results
_print_sync_stats(stats)
# Exit with error if there were failures
if stats.files_failed > 0:
sys.exit(1)
def run_sync_local(args):
"""Run sync operation for local filesystem."""
# Validate path
local_path = args.path
if not local_path:
print("ERROR: --path is required")
sys.exit(1)
# Load config (for GCS and chunking settings)
config = load_ingestion_config()
# Initialize local file source
print(f"Initializing local filesystem source: {local_path}")
local_source = LocalFileSource(root_path=local_path)
if not local_source.authenticate():
print(f"ERROR: Failed to access local path: {local_path}")
sys.exit(1)
print(" Local filesystem authenticated successfully")
# Initialize storage backend
use_noop_storage = getattr(args, "no_storage", False)
storage: StorageBackend
if use_noop_storage:
print("Initializing no-op storage (files will NOT be persisted)...")
storage = NoOpStorageBackend(prefix="noop://local/")
print(" Storage: no-op (dry run)")
else:
print("Initializing GCS storage...")
storage = GCSStorageBackend(
bucket_name=config.gcs.bucket_name,
project_id=config.gcs.project_id,
credentials_path=config.gcs.credentials_path,
prefix=config.gcs.prefix,
)
print(f" GCS bucket: gs://{config.gcs.bucket_name}/{config.gcs.prefix}")
# Initialize KG client
print("Connecting to Supabase...")
kg = get_kg_client()
print(" Supabase connected successfully")
# Determine what processing to enable
enable_chunking = not getattr(args, "no_chunks", False)
enable_claims = config.claim_extraction.enabled and not getattr(args, "no_claims", False)
# Show processing settings
print()
print("Processing settings:")
print(f" Chunking: {'enabled' if enable_chunking else 'DISABLED (--no-chunks)'}")
print(f" Claim extraction: {'enabled' if enable_claims else 'disabled'}")
if enable_claims:
print(f" Provider: {config.claim_extraction.llm_provider}")
print(f" Model: {config.claim_extraction.model}")
# Get company folder depth (use same config as gdrive, or override via args)
company_folder_depth = (
getattr(args, "company_depth", None) or config.gdrive.company_folder_depth
)
# Get company override if specified
override_company = getattr(args, "company", None)
if override_company:
print(f" Company override: {override_company}")
# Create orchestrator
orchestrator = IngestionOrchestrator(
source=local_source,
storage=storage,
kg_client=kg,
config=IngestionConfig(
root_folder_id=str(local_source.root_path),
company_folder_depth=company_folder_depth,
chunk_size=config.chunking.chunk_size,
chunk_overlap=config.chunking.chunk_overlap,
batch_size=config.batch_size,
max_file_size_mb=config.max_file_size_mb,
enable_chunking=enable_chunking,
enable_claim_extraction=enable_claims,
claim_extraction_llm=config.claim_extraction.llm_provider,
claim_extraction_model=config.claim_extraction.model,
override_company_name=override_company,
),
)
# Run sync
print()
print("=" * 60)
print(f"Running LOCAL SYNC: {local_path}")
print("=" * 60)
stats = orchestrator.run_full_sync()
# Print results
_print_sync_stats(stats)
# Print storage stats if using no-op storage
if use_noop_storage and isinstance(storage, NoOpStorageBackend):
storage.print_summary()
# Exit with error if there were failures
if stats.files_failed > 0:
sys.exit(1)
def _print_sync_stats(stats):
"""Print detailed sync statistics."""
print()
print("=" * 70)
print("SYNC COMPLETED")
print("=" * 70)
# Overall summary
print()
print("SUMMARY")
print("-" * 70)
print(f" Files found: {stats.files_found}")
print(f" Files processed: {stats.files_processed}")
print(f" Files skipped: {stats.files_skipped}")
print(f" Files failed: {stats.files_failed}")
print(f" Chunks created: {stats.chunks_created}")
print(f" Claims extracted: {stats.claims_extracted}")
# Group documents by company
docs_by_company: dict[str, list] = {}
for doc in stats.documents:
company = doc.company_name or "(No Company)"
if company not in docs_by_company:
docs_by_company[company] = []
docs_by_company[company].append(doc)
# Per-company breakdown with nested documents
if stats.companies or docs_by_company:
print()
print("BY COMPANY")
print("-" * 70)
for company_name in sorted(docs_by_company.keys()):
company_docs = docs_by_company[company_name]
company_stat = stats.companies.get(company_name)
# Company header with stats
processed = [d for d in company_docs if d.status == "processed"]
skipped = [d for d in company_docs if d.status == "skipped"]
failed = [d for d in company_docs if d.status == "failed"]
print(f"\n {company_name}")
print(
f" Files: {len(company_docs)} total, "
f"{len(processed)} processed, "
f"{len(skipped)} skipped, "
f"{len(failed)} failed"
)
if company_stat and (company_stat.chunks_created or company_stat.claims_extracted):
print(
f" Output: {company_stat.chunks_created} chunks, "
f"{company_stat.claims_extracted} claims"
)
# Processed documents
if processed:
print()
print(" Processed:")
for doc in processed:
print(f" - {doc.name}")
print(f" ID: {doc.document_id}")
print(f" {doc.chunks_created} chunks, {doc.claims_extracted} claims")
# Failed documents
if failed:
print()
print(" Failed:")
for doc in failed:
print(f" - {doc.name}")
if doc.document_id:
print(f" ID: {doc.document_id}")
print(f" Error: {doc.error}")
# Skipped documents (grouped by reason)
if skipped:
print()
print(" Skipped:")
by_reason: dict[str, list] = {}
for doc in skipped:
reason = doc.skip_reason or "unknown"
if reason not in by_reason:
by_reason[reason] = []
by_reason[reason].append(doc)
for reason, docs in sorted(by_reason.items()):
if len(docs) <= 3:
for doc in docs:
print(f" - {doc.name} ({reason})")
else:
print(f" - {len(docs)} files ({reason})")
if stats.errors:
print()
print("ERRORS")
print("-" * 70)
for error in stats.errors[:10]:
print(f" - {error}")
if len(stats.errors) > 10:
print(f" ... and {len(stats.errors) - 10} more errors")
def show_status(args):
"""Show status of recent ingestion jobs."""
print("Connecting to Supabase...")
kg = get_kg_client()
# Query recent jobs
resp = (
kg.client.table("ingestion_jobs")
.select("*")
.order("created_at", desc=True)
.limit(10)
.execute()
)
if not resp.data:
print("\nNo ingestion jobs found.")
return
print("\nRecent Ingestion Jobs:")
print("-" * 80)
for job in resp.data:
created = job.get("created_at", "N/A")[:19] if job.get("created_at") else "N/A"
status = job.get("status", "N/A")
job_type = job.get("job_type", "N/A")
source = job.get("source_type", "N/A")
status_icon = {"succeeded": "✓", "failed": "✗", "running": "⟳", "queued": "○"}.get(
status, "?"
)
print(f"{status_icon} [{status:10}] {job_type:20} ({source}) - {created}")
# Show payload summary if available
payload = job.get("payload", {})
if isinstance(payload, dict):
if "files_processed" in payload:
print(
f" Processed: {payload.get('files_processed', 0)}, "
f"Skipped: {payload.get('files_skipped', 0)}, "
f"Failed: {payload.get('files_failed', 0)}"
)
if job.get("error_message"):
print(f" Error: {job['error_message'][:100]}")
print("-" * 80)
SCRAPER_REGISTRY = {
"youtube": "YouTubeScraper",
}
def run_process(args):
"""Process existing documents to generate chunks and/or claims."""
# Load config
config = load_ingestion_config()
# Initialize KG client
print("Connecting to Supabase...")
kg = get_kg_client()
print(" Supabase connected successfully")
# Show settings
print()
print("Processing settings:")
print(f" Chunking: {'enabled' if args.chunks else 'disabled'}")
print(f" Claims: {'enabled' if args.claims else 'disabled'}")
if args.claims:
print(f" Provider: {config.claim_extraction.llm_provider}")
print(f" Model: {config.claim_extraction.model}")
if not args.chunks and not args.claims:
print("\nERROR: Must specify --chunks and/or --claims")
sys.exit(1)
# Build document query
documents = []
if args.document_id:
# Process specific document
doc = kg.get_document_by_id(args.document_id)
if not doc:
print(f"ERROR: Document not found: {args.document_id}")
sys.exit(1)
documents = [doc]
elif args.company:
# Process documents for a company
company = kg.get_company_by_name(args.company)
if not company:
print(f"ERROR: Company not found: {args.company}")
sys.exit(1)
documents = kg.get_documents_for_company(company.id, limit=args.limit)
elif args.missing:
# Process documents missing chunks or claims
documents = _get_documents_missing_processing(kg, args.missing, args.limit)
else:
print("ERROR: Must specify --document-id, --company, or --missing")
sys.exit(1)
if not documents:
print("\nNo documents found to process")
return
print(f"\nDocuments to process: {len(documents)}")
# Initialize processors
from hashlib import sha256
from llama_index.core import Document as LlamaDoc
from llama_index.core.node_parser import SentenceSplitter
from llama_index.embeddings.openai import OpenAIEmbedding
from models.internal import SourceType
from services.claims import ClaimExtractor
from services.llm import GeminiLLM, OpenAiLLM
claim_extractor = None
if args.claims:
if config.claim_extraction.llm_provider == "openai":
api_key = config.llm.openai_api_key
if not api_key:
print("ERROR: OPENAI_API_KEY required for claim extraction")
sys.exit(1)
llm = OpenAiLLM(api_key=api_key, model_name=config.claim_extraction.model)
else:
api_key = config.llm.gemini_api_key
if not api_key:
print("ERROR: GEMINI_API_KEY required for claim extraction")
sys.exit(1)
llm = GeminiLLM(api_key=api_key, model_name=config.claim_extraction.model)
claim_extractor = ClaimExtractor(llm=llm)
embed_model = None
splitter = None
if args.chunks:
embed_model = OpenAIEmbedding(model="text-embedding-3-small")
splitter = SentenceSplitter(
chunk_size=config.chunking.chunk_size,
chunk_overlap=config.chunking.chunk_overlap,
)
# Create job for tracking
job_id = kg.create_ingestion_job(
job_type="process",
source_type="existing_documents",
payload={
"chunks_enabled": args.chunks,
"claims_enabled": args.claims,
"document_count": len(documents),
},
)
# Process documents
total_processed = 0
total_skipped = 0
total_failed = 0
total_chunks = 0
total_claims = 0
for doc in documents:
print(f"\nProcessing: {doc.title or doc.id}")
if not doc.parsed_text:
print(" Skipped: no parsed_text")
total_skipped += 1
continue
try:
# Generate chunks
if args.chunks and splitter and embed_model:
# Delete existing chunks if re-processing
if args.force:
kg.delete_chunks_for_document(doc.id)
llama_doc = LlamaDoc(text=doc.parsed_text)
nodes = splitter.get_nodes_from_documents([llama_doc])
if nodes:
chunks_to_insert = []
for idx, node in enumerate(nodes):
chunk_text = node.text
chunk_hash = sha256(chunk_text.encode()).hexdigest()
embedding = embed_model.get_text_embedding(chunk_text)
chunks_to_insert.append(
{
"document_id": doc.id,
"company_id": doc.company_id,
"chunk_index": idx,
"chunk_text": chunk_text,
"chunk_hash": chunk_hash,
"embedding": embedding,
"chunk_kind": "document_section",
}
)
kg.create_chunks_batch(chunks_to_insert)
total_chunks += len(chunks_to_insert)
print(f" Created {len(chunks_to_insert)} chunks")
# Extract claims
if args.claims and claim_extractor:
# Delete existing claims if re-processing
if args.force:
kg.delete_claims_for_document(doc.id)
# Infer source type from document
source_type = (
SourceType(doc.source_type) if doc.source_type else SourceType.UNDEFINED
)
claims = claim_extractor.extract(
text=doc.parsed_text,
source_type=source_type,
document_id=doc.id,
)
if claims:
kg_claims = claim_extractor.to_kg_format(
claims=claims,
document_id=doc.id,
company_id=doc.company_id,
)
count = kg.create_claims_batch(kg_claims)
total_claims += count
print(f" Extracted {count} claims")
total_processed += 1
except Exception as e:
total_failed += 1
print(f" Error: {e}")
logger.exception(f"Failed to process document {doc.id}")
# Update job
kg.update_ingestion_job(
job_id,
status="succeeded" if total_failed == 0 else "completed_with_errors",
payload={
"documents_processed": total_processed,
"documents_skipped": total_skipped,
"documents_failed": total_failed,
"chunks_created": total_chunks,
"claims_extracted": total_claims,
},
)
# Summary
print()
print("=" * 60)
print("PROCESSING COMPLETED")
print("=" * 60)
print(f" Documents processed: {total_processed}")
print(f" Documents skipped: {total_skipped}")
print(f" Documents failed: {total_failed}")
print(f" Chunks created: {total_chunks}")
print(f" Claims extracted: {total_claims}")
def _get_documents_missing_processing(kg: KGClient, missing_type: str, limit: int) -> list:
"""Get documents missing chunks or claims."""
if missing_type == "chunks":
# Documents with parsed_text but no chunks
resp = kg.client.rpc("get_documents_without_chunks", {"p_limit": limit}).execute()
elif missing_type == "claims":
# Documents with parsed_text but no claims
resp = kg.client.rpc("get_documents_without_claims", {"p_limit": limit}).execute()
else:
# Try raw query as fallback
resp = (
kg.client.table("documents")
.select("*")
.not_.is_("parsed_text", "null")
.limit(limit)
.execute()
)
return [kg._row_to_document(row) for row in resp.data]
def _build_scraper(source: str, scraping_config, kg) -> ContentSource:
"""Build scraper instance for the given source."""
if source == "youtube":
if not scraping_config.youtube_api_key:
print("ERROR: YOUTUBE_API_KEY environment variable required")
sys.exit(1)
from services.scrapers.youtube import YouTubeScraper
return YouTubeScraper(scraping_config.youtube_api_key, kg, scraping_config.youtube)
raise ValueError(f"Unknown source: {source}")
def run_scrape(args):
"""Run scrape operation for public sources using the full ingestion pipeline."""
if not args.company and not args.companies:
print("ERROR: Must specify --company or --companies all")
sys.exit(1)
# Load configs
config = load_ingestion_config()
scraping_config = load_scraping_config()
# Initialize KG client
print("Connecting to Supabase...")
kg = get_kg_client()
print(" Supabase connected successfully")
# Build scraper
print(f"Initializing {args.source} scraper...")
scraper = _build_scraper(args.source, scraping_config, kg)
if not scraper.authenticate():
print(f"ERROR: Failed to authenticate {args.source}")
sys.exit(1)
print(f" {args.source} authenticated successfully")
# Initialize GCS storage for pipeline
print("Initializing GCS storage...")
gcs = GCSStorageBackend(
bucket_name=config.gcs.bucket_name,
project_id=config.gcs.project_id,
credentials_path=config.gcs.credentials_path,
prefix=config.gcs.prefix,
)
print(f" GCS bucket: gs://{config.gcs.bucket_name}/{config.gcs.prefix}")
# Show claim extraction settings
print()
print("Claim extraction:")
print(f" Enabled: {config.claim_extraction.enabled}")
if config.claim_extraction.enabled:
print(f" Provider: {config.claim_extraction.llm_provider}")
print(f" Model: {config.claim_extraction.model}")
# Get target companies
if args.company:
company_obj = kg.get_company_by_name(args.company)
if not company_obj:
print(f"ERROR: Company not found: {args.company}")
sys.exit(1)
target_companies = [company_obj]
else:
# Get all companies from KG
resp = kg.client.table("companies").select("*").execute()
from services.kg import KGCompany
target_companies = [
KGCompany(
id=row["id"],
name=row["name"],
normalized_name=row["normalized_name"],
domain=row.get("domain"),
aliases=row.get("aliases"),
external_ids=row.get("external_ids"),
metadata=row.get("metadata"),
)
for row in resp.data
]
if not target_companies:
print("No companies found to scrape")
return
print(f"\nTarget companies: {len(target_companies)}")
# Track detailed stats
total_found = 0
total_processed = 0
total_skipped = 0
total_chunks = 0
total_claims = 0
total_failed = 0
# Per-company and per-document tracking
company_stats: dict[str, dict] = {}
processed_docs: list[dict] = []
failed_docs: list[dict] = []
skipped_docs: list[dict] = []
for company in target_companies:
print(f"\nScraping {args.source} for: {company.name}")
items = list(scraper.list_folder(company.id))
company_found = len(items)
total_found += company_found
print(f" Found {company_found} items")
# Initialize company stats
company_stats[company.name] = {
"found": company_found,
"processed": 0,
"skipped": 0,
"failed": 0,
"chunks": 0,
"claims": 0,
}
if args.dry_run:
for item in items[: args.limit]:
print(f" Would scrape: {item.name}")
if hasattr(item, "metadata") and item.metadata.get("url"):
print(f" URL: {item.metadata['url']}")
else:
# Create orchestrator for this company's batch
orchestrator = IngestionOrchestrator(
source=scraper,
storage=gcs,
kg_client=kg,
config=IngestionConfig(
root_folder_id=company.id,
company_folder_depth=1,
chunk_size=config.chunking.chunk_size,
chunk_overlap=config.chunking.chunk_overlap,
batch_size=config.batch_size,
max_file_size_mb=config.max_file_size_mb,
enable_claim_extraction=config.claim_extraction.enabled,
claim_extraction_llm=config.claim_extraction.llm_provider,
claim_extraction_model=config.claim_extraction.model,
),
)
# Create ingestion job for tracking
job_id = kg.create_ingestion_job(
job_type="scrape",
source_type=scraper.source_type,
payload={"company_id": company.id, "company_name": company.name},
)
for item in items[: args.limit]:
print(f" Processing: {item.name}")
try:
result = orchestrator._process_file(item, job_id)
if result.skipped:
reason = result.skip_reason or "unknown"
print(f" Skipped: {reason}")
total_skipped += 1
company_stats[company.name]["skipped"] += 1
skipped_docs.append(
{
"name": item.name,
"company": company.name,
"reason": reason,
}
)
elif result.success:
total_processed += 1
total_chunks += result.chunk_count
total_claims += result.claim_count
company_stats[company.name]["processed"] += 1
company_stats[company.name]["chunks"] += result.chunk_count
company_stats[company.name]["claims"] += result.claim_count
print(f" Done: {result.chunk_count} chunks, {result.claim_count} claims")
processed_docs.append(
{
"name": item.name,
"company": company.name,
"document_id": result.document_id,
"chunks": result.chunk_count,
"claims": result.claim_count,
}
)
else:
total_failed += 1
company_stats[company.name]["failed"] += 1
print(f" Failed: {result.error}")
failed_docs.append(
{
"name": item.name,
"company": company.name,
"error": result.error,
}
)
except Exception as e:
total_failed += 1
company_stats[company.name]["failed"] += 1
print(f" Error: {e}")
failed_docs.append(
{
"name": item.name,
"company": company.name,
"error": str(e),
}
)
# Update job status
kg.update_ingestion_job(
job_id,
status="succeeded"
if company_stats[company.name]["failed"] == 0
else "completed_with_errors",
payload={
"company_id": company.id,
"company_name": company.name,
"files_found": company_found,
"files_processed": company_stats[company.name]["processed"],
"files_skipped": company_stats[company.name]["skipped"],
"files_failed": company_stats[company.name]["failed"],
"chunks_created": company_stats[company.name]["chunks"],
"claims_extracted": company_stats[company.name]["claims"],
},
)
# Print detailed summary
_print_scrape_stats(
args,
total_found,
total_processed,
total_skipped,
total_failed,
total_chunks,
total_claims,
company_stats,
processed_docs,
failed_docs,
skipped_docs,
)
def _print_scrape_stats(
args,
total_found,
total_processed,
total_skipped,
total_failed,
total_chunks,
total_claims,
company_stats,
processed_docs,
failed_docs,
skipped_docs,
):
"""Print detailed scrape statistics."""
print()
print("=" * 70)
print("SCRAPE COMPLETED")
print("=" * 70)
# Overall summary
print()
print("SUMMARY")
print("-" * 70)
print(f" Items found: {total_found}")
if not args.dry_run:
print(f" Items processed: {total_processed}")
print(f" Items skipped: {total_skipped}")
print(f" Items failed: {total_failed}")
print(f" Chunks created: {total_chunks}")
print(f" Claims extracted: {total_claims}")
if args.dry_run:
return
# Per-company breakdown with nested documents
if company_stats:
print()
print("BY COMPANY")
print("-" * 70)
for company_name, stats in sorted(company_stats.items()):
# Company header
print(f"\n {company_name}")
print(
f" Items: {stats['found']} found, "
f"{stats['processed']} processed, "
f"{stats['skipped']} skipped, "
f"{stats['failed']} failed"
)
if stats["chunks"] or stats["claims"]:
print(f" Output: {stats['chunks']} chunks, {stats['claims']} claims")
# Processed documents for this company
company_processed = [d for d in processed_docs if d["company"] == company_name]
if company_processed:
print()
print(" Processed:")
for doc in company_processed:
print(f" - {doc['name']}")
print(f" ID: {doc['document_id']}")
print(f" {doc['chunks']} chunks, {doc['claims']} claims")
# Failed documents for this company
company_failed = [d for d in failed_docs if d["company"] == company_name]
if company_failed:
print()
print(" Failed:")
for doc in company_failed:
print(f" - {doc['name']}")
print(f" Error: {doc['error']}")
# Skipped documents for this company
company_skipped = [d for d in skipped_docs if d["company"] == company_name]
if company_skipped:
print()
print(" Skipped:")
# Group by reason
by_reason: dict[str, list] = {}
for doc in company_skipped:
reason = doc["reason"]
if reason not in by_reason:
by_reason[reason] = []
by_reason[reason].append(doc)
for reason, docs in sorted(by_reason.items()):
if len(docs) <= 3:
for doc in docs:
print(f" - {doc['name']} ({reason})")
else:
print(f" - {len(docs)} items ({reason})")
def main():
parser = argparse.ArgumentParser(
description="Knowledge Graph Ingestion CLI",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog=__doc__,
)
subparsers = parser.add_subparsers(dest="command")
# Sync command (Google Drive)
sync_parser = subparsers.add_parser("sync", help="Run full sync of all files from Google Drive")
sync_parser.add_argument(
"--no-chunks", action="store_true", help="Skip chunking and embedding generation"
)
sync_parser.add_argument("--no-claims", action="store_true", help="Skip claim extraction")
sync_parser.add_argument(
"--company",
type=str,
help="Override company for all documents (creates company if doesn't exist)",
)
# Sync-local command (Local filesystem)
sync_local_parser = subparsers.add_parser(
"sync-local", help="Run full sync of files from local filesystem"
)
sync_local_parser.add_argument(
"--path", type=str, required=True, help="Path to local directory to ingest"
)
sync_local_parser.add_argument(
"--no-chunks", action="store_true", help="Skip chunking and embedding generation"
)
sync_local_parser.add_argument("--no-claims", action="store_true", help="Skip claim extraction")
sync_local_parser.add_argument(
"--company-depth",
type=int,
help="Folder depth where company names appear (default: uses GDRIVE_COMPANY_FOLDER_DEPTH)",
)
sync_local_parser.add_argument(
"--no-storage",
action="store_true",
help="Skip uploading files to GCS (useful for testing)",
)
sync_local_parser.add_argument(
"--company",
type=str,
help="Override company for all documents (creates company if doesn't exist)",
)
# Scrape command
scrape_parser = subparsers.add_parser("scrape", help="Scrape public sources for content")
scrape_parser.add_argument(
"--source",
type=str,
choices=list(SCRAPER_REGISTRY.keys()),
required=True,
help="Source to scrape",
)
scrape_parser.add_argument("--company", type=str, help="Specific company name")
scrape_parser.add_argument(
"--companies", type=str, choices=["all"], help="All tracked companies"
)
scrape_parser.add_argument("--limit", type=int, default=20, help="Max items per company")
scrape_parser.add_argument("--dry-run", action="store_true", help="Show what would be scraped")
# Process command
process_parser = subparsers.add_parser(
"process", help="Generate chunks and/or claims from existing documents"
)
process_parser.add_argument("--document-id", type=str, help="Process a specific document by ID")
process_parser.add_argument("--company", type=str, help="Process all documents for a company")
process_parser.add_argument(
"--missing",
type=str,
choices=["chunks", "claims"],
help="Process documents missing chunks or claims",
)
process_parser.add_argument(
"--chunks", action="store_true", help="Generate chunks with embeddings"
)
process_parser.add_argument(
"--claims", action="store_true", help="Extract claims from documents"
)
process_parser.add_argument(
"--force",
action="store_true",
help="Delete existing chunks/claims before re-processing",
)
process_parser.add_argument("--limit", type=int, default=100, help="Max documents to process")
# Validate command
validate_parser = subparsers.add_parser("validate", help="Validate configuration")
# Status command
status_parser = subparsers.add_parser("status", help="Check status of recent ingestion jobs")
args = parser.parse_args()
if args.command == "sync":
run_sync(args)
elif args.command == "sync-local":
run_sync_local(args)
elif args.command == "scrape":
run_scrape(args)
elif args.command == "process":
run_process(args)
elif args.command == "validate":
if not validate_config():
sys.exit(1)
elif args.command == "status":
show_status(args)
else:
parser.print_help()
sys.exit(1)
if __name__ == "__main__":
main()
#!/usr/bin/env python3
"""
CLI for running ingestion operations.
Usage:
python -m cli.ingest sync
python -m cli.ingest scrape --source youtube --company "Anthropic"
python -m cli.ingest process --company "Anthropic" --chunks --claims
python -m cli.ingest status
python -m cli.ingest validate
Examples:
# Run a full sync of all files in the configured Google Drive folder
python -m cli.ingest sync
# Scrape YouTube for a specific company
python -m cli.ingest scrape --source youtube --company "Anthropic"
# Scrape YouTube for all tracked companies (dry run)
python -m cli.ingest scrape --source youtube --companies all --dry-run
# Generate chunks and claims for a company's documents
python -m cli.ingest process --company "Anthropic" --chunks --claims
# Re-process a specific document (delete existing and regenerate)
python -m cli.ingest process --document-id "uuid" --chunks --claims --force
# Process documents missing claims
python -m cli.ingest process --missing claims --claims --limit 50
# Validate configuration without running sync
python -m cli.ingest validate
# Check status of recent ingestion jobs
python -m cli.ingest status
"""
import argparse
import logging
import sys
from config.ingestion import load_ingestion_config
from config.scraping import load_scraping_config
from services.ingestion.orchestrator import IngestionConfig, IngestionOrchestrator
from services.kg import KGClient, get_kg_client
from services.sources.base import ContentSource
from services.sources.gdrive import GoogleDriveSource
from services.sources.local import LocalFileSource
from services.storage.base import StorageBackend
from services.storage.gcs import GCSStorageBackend
from services.storage.noop import NoOpStorageBackend
# Configure logging
logging.basicConfig(
level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
)
logger = logging.getLogger(__name__)
def validate_config():
"""Validate configuration and return errors."""
config = load_ingestion_config()
errors = config.validate()
if errors:
print("Configuration errors:")
for error in errors:
print(f" - {error}")
return False
print("Configuration is valid:")
print(f" - Google Drive folder: {config.gdrive.root_folder_id}")
print(f" - GCS bucket: {config.gcs.bucket_name}")
print(f" - Company folder depth: {config.gdrive.company_folder_depth}")
print(f" - Chunk size: {config.chunking.chunk_size}")
print()
print("Claim extraction settings:")
print(f" - Enabled: {config.claim_extraction.enabled}")
print(f" - LLM provider: {config.claim_extraction.llm_provider}")
print(f" - Model: {config.claim_extraction.model}")
return True
def run_sync(args):
"""Run sync operation."""
# Load and validate config
config = load_ingestion_config()
errors = config.validate()
if errors:
print("Configuration errors:")
for error in errors:
print(f" - {error}")
sys.exit(1)
# Initialize Google Drive source
print("Initializing Google Drive connection...")
gdrive = GoogleDriveSource(
credentials_path=config.gdrive.credentials_path,
credentials_dict=config.gdrive.get_credentials_dict(),
)