Buckets:

download
raw
38.3 kB
#!/usr/bin/env python3 """Nancy Brain CLI interface.""" import click import json import logging import os import sys import subprocess import yaml from pathlib import Path # Avoid OpenMP duplicate runtime crashes by allowing duplicate lib loading when not set externally. # Respect an existing environment setting (e.g., in a .env) but default to TRUE for CLI runs. os.environ.setdefault("KMP_DUPLICATE_LIB_OK", "TRUE") # Optional rich integration for colored output and spinners try: from rich.console import Console from rich.spinner import Spinner from rich.tree import Tree from rich.panel import Panel from rich.table import Table from rich.text import Text RICH_AVAILABLE = True _console = Console() except Exception: RICH_AVAILABLE = False _console = None # Add the package root to sys.path to handle relative imports package_root = Path(__file__).parent.parent sys.path.insert(0, str(package_root)) from nancy_brain.config_validation import ( validate_repositories_config, validate_articles_config, ) # Get version from package try: from nancy_brain import __version__ except ImportError: __version__ = "unknown" @click.group() @click.version_option(version=__version__) def cli(): """Nancy Brain - Turn GitHub repos into AI-searchable knowledge bases.""" pass def _configure_cli_logging() -> None: """Ensure OCR commands emit INFO logs to stderr for batch-job observability.""" root_logger = logging.getLogger() if not root_logger.handlers: logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") else: root_logger.setLevel(logging.INFO) @cli.command() @click.argument("project_name") def init(project_name): """Initialize a new Nancy Brain project. This command creates a minimal `config/` directory with a `repositories.yml` file to get you started. Edit the file and then run `nancy-brain build`. """ project_path = Path(project_name) project_path.mkdir(exist_ok=True) # Create basic config structure config_dir = project_path / "config" config_dir.mkdir(exist_ok=True) # Basic repositories.yml repos_config = config_dir / "repositories.yml" repos_config.write_text( """# Add your repositories here # example_tools: # - name: example-repo # url: https://github.com/org/example-repo.git """ ) click.echo(f"โœ… Initialized Nancy Brain project in {project_name}/") click.echo(f"๐Ÿ“ Edit {repos_config} to add repositories") click.echo("๐Ÿ—๏ธ Run 'nancy-brain build' to create the knowledge base") @cli.command() @click.option("--config", default="config/repositories.yml", help="Repository config file") @click.option("--articles-config", help="PDF articles config file") @click.option( "--embeddings-path", default="knowledge_base/embeddings", help="Embeddings output path", ) @click.option("--force-update", is_flag=True, help="Force update all repositories") @click.option("--dry-run", is_flag=True, help="Show what would be done without executing the build") @click.option("--dirty", is_flag=True, help="Leave raw repos and PDFs in place after build (don't cleanup)") @click.option( "--summaries/--no-summaries", default=None, help="Generate document summaries during build (local, custom OpenAI-compatible, or Anthropic)", ) @click.option( "--batch-size", default=0, type=int, help="Index documents in batches (requires embeddings.upsert support). 0 = disable batching.", ) @click.option( "--max-docs", default=0, type=int, help="Stop after indexing this many document chunks (for testing / limiting resource use). 0 = no limit.", ) @click.option( "--category", help="Limit build to a single repository category (as defined in repositories.yml)", ) @click.option( "--repo", default=None, help="Limit build to a single repository by name (across all categories unless --category is set).", ) @click.option( "--summaries-only", is_flag=True, default=False, help="Warm the summary cache per-repo then exit without building the index. Pair with --repo for one-node-per-repo cluster jobs.", ) @click.option( "--use-cached-ocr-only", is_flag=True, default=False, help="Only use cached OCR Markdown for PDFs. Missing cache entries are deferred as needs_ocr.", ) def build( config, articles_config, embeddings_path, force_update, dry_run, dirty, summaries, batch_size, max_docs, category, repo, summaries_only, use_cached_ocr_only, ): """Build the knowledge base from configured repositories. The build command validates `config/repositories.yml` (and `config/articles.yml` if provided) before starting. If validation fails, the command prints detailed errors and exits with a non-zero status. """ click.echo("๐Ÿ—๏ธ Building knowledge base...") # Convert paths to absolute paths relative to current working directory config_path = Path.cwd() / config embeddings_path = Path.cwd() / embeddings_path # Pre-validate repository config to provide immediate feedback if not config_path.exists(): click.echo(f"โŒ Repository config not found: {config_path}") sys.exit(2) try: with open(config_path, "r", encoding="utf-8") as f: cfg = yaml.safe_load(f) or {} except Exception as e: click.echo(f"โŒ Failed to read repository config: {e}") sys.exit(2) ok, errors = validate_repositories_config(cfg) if not ok: click.echo("โŒ repositories.yml validation failed:") for err in errors: click.echo(f" - {err}") sys.exit(2) # Build command arguments cmd = [ sys.executable, str(package_root / "scripts" / "build_knowledge_base.py"), "--config", str(config_path), "--embeddings-path", str(embeddings_path), ] if articles_config: articles_config_path = Path.cwd() / articles_config # Validate articles config as well if not articles_config_path.exists(): click.echo(f"โŒ Articles config not found: {articles_config_path}") sys.exit(2) try: with open(articles_config_path, "r", encoding="utf-8") as f: a_cfg = yaml.safe_load(f) or {} except Exception as e: click.echo(f"โŒ Failed to read articles config: {e}") sys.exit(2) ok2, errs2 = validate_articles_config(a_cfg) if not ok2: click.echo("โŒ articles.yml validation failed:") for err in errs2: click.echo(f" - {err}") sys.exit(2) cmd.extend(["--articles-config", str(articles_config_path)]) if force_update: cmd.append("--force-update") if dirty: cmd.append("--dirty") if summaries is True: cmd.append("--summaries") elif summaries is False: cmd.append("--no-summaries") if batch_size and batch_size > 0: cmd.extend(["--batch-size", str(batch_size)]) if max_docs and max_docs > 0: cmd.extend(["--max-docs", str(max_docs)]) if category: cmd.extend(["--category", category]) if repo: cmd.extend(["--repo", repo]) if summaries_only: cmd.append("--summaries-only") if use_cached_ocr_only: cmd.append("--use-cached-ocr-only") # If dry-run requested, still run the underlying script with --dry-run so that # repository cloning/downloading/indexing intentions and validation summaries # are produced by the central pipeline logic rather than a hollow preview. if dry_run: cmd.append("--dry-run") if RICH_AVAILABLE: _console.print("[yellow]๐Ÿ”Ž Dry run: executing pipeline in no-op mode[/yellow]") else: click.echo(click.style("๐Ÿ”Ž Dry run: executing pipeline in no-op mode", fg="yellow")) # Run the build script from the package directory try: if RICH_AVAILABLE: with _console.status("Building knowledge base...", spinner="dots"): result = subprocess.run(cmd, check=True) else: result = subprocess.run(cmd, check=True) success_msg = "โœ… Knowledge base built successfully!" if RICH_AVAILABLE: _console.print(f"[green]{success_msg}[/green]") else: click.echo(click.style(success_msg, fg="green")) except subprocess.CalledProcessError as e: err_msg = f"โŒ Build failed with exit code {e.returncode}" if RICH_AVAILABLE: _console.print(f"[red]{err_msg}[/red]") else: click.echo(click.style(err_msg, fg="red")) sys.exit(e.returncode) @cli.group() def ocr(): """OCR utilities and cache warmers.""" pass @ocr.command("warm") @click.argument("paths", nargs=-1, type=click.Path(path_type=Path)) @click.option( "--cache-dir", type=click.Path(path_type=Path), default=None, help="OCR cache directory (defaults to the package cache path).", ) @click.option( "--backend", default=None, help="Preferred OCR backend (for example deepseek, skip, or none).", ) @click.option( "--articles-config", type=click.Path(path_type=Path), default=None, help="Optional articles.yml to download before warming OCR cache.", ) @click.option( "--base-path", type=click.Path(path_type=Path), default=Path("knowledge_base/raw"), show_default=True, help="Base path for downloaded PDFs and scan roots.", ) @click.option( "--recursive/--no-recursive", default=True, help="Recurse into directories when scanning for PDFs.", ) def ocr_warm(paths, cache_dir, backend, articles_config, base_path, recursive): """Warm OCR cache entries for PDFs.""" from nancy_brain.pdf_ocr import warm_pdf_ocr_cache _configure_cli_logging() scan_paths = list(paths) if articles_config: from scripts.build_knowledge_base import download_pdf_articles download_pdf_articles(str(articles_config), base_path=str(base_path)) article_paths = _existing_article_pdf_paths(articles_config, base_path) scan_paths.extend(article_paths) if not scan_paths: if articles_config: click.echo(f"No article PDFs found under {base_path} for config {articles_config}") sys.exit(1) scan_paths = [base_path] results = warm_pdf_ocr_cache(scan_paths, cache_dir=cache_dir, preferred_backend=backend, recursive=recursive) generated = sum(1 for result in results if result.status == "generated") cached = sum(1 for result in results if result.cached) deferred = sum(1 for result in results if result.needs_ocr) failed = sum(1 for result in results if result.status == "error") click.echo(f"PDFs scanned: {len(results)}") click.echo(f"Cached hits: {cached}") click.echo(f"Generated: {generated}") click.echo(f"Deferred (needs OCR): {deferred}") click.echo(f"Failed: {failed}") for result in results: if result.needs_ocr and result.warning: click.echo(f" - needs_ocr: {result.warning}") elif result.status == "error" and result.warning: click.echo(f" - error: {result.warning}") if failed or ((generated + cached) == 0 and deferred > 0): if failed == 0: click.echo("No OCR cache entries were generated or reused; all PDFs were deferred for OCR.") sys.exit(1) def _existing_article_pdf_paths(articles_config: Path, base_path: Path) -> list[Path]: """Return existing article PDFs declared in the provided articles config.""" with open(articles_config, "r", encoding="utf-8") as handle: config = yaml.safe_load(handle) or {} seen: set[Path] = set() article_paths: list[Path] = [] for category, articles in config.items(): if not isinstance(articles, list): continue for article in articles: if not isinstance(article, dict): continue article_name = str(article.get("name", "")).strip() if not article_name: continue pdf_path = (base_path / category / f"{article_name}.pdf").resolve(strict=False) if pdf_path.exists() and pdf_path not in seen: article_paths.append(pdf_path) seen.add(pdf_path) return article_paths @ocr.command("setup") @click.option( "--shared-root", type=click.Path(path_type=Path), default=None, help="Override the managed shared worker install root.", ) @click.option( "--python", "python_executable", type=click.Path(path_type=Path), default=None, help="Python executable used to create the worker venv (defaults to the current interpreter).", ) @click.option( "--torch-index-url", default=None, help="Optional PyTorch wheel index URL for the isolated worker runtime.", ) @click.option( "--torch-version", default=None, help="Optional torch version override for the worker runtime.", ) @click.option( "--torchvision-version", default=None, help="Optional torchvision version override for the worker runtime.", ) @click.option( "--flash-attn/--no-flash-attn", default=False, help="Attempt to install flash-attn into the worker runtime.", ) @click.option( "--recreate", is_flag=True, default=False, help="Recreate the worker virtualenv from scratch before installing packages.", ) @click.option( "--verify/--no-verify", default=True, help="Verify the installed worker runtime after setup.", ) def ocr_setup( shared_root, python_executable, torch_index_url, torch_version, torchvision_version, flash_attn, recreate, verify, ): """Install or update the managed local OCR worker runtime.""" from nancy_brain.ocr_worker_runtime import install_local_ocr_worker try: summary = install_local_ocr_worker( root=shared_root, python_executable=python_executable, torch_index_url=torch_index_url, torch_version=torch_version, torchvision_version=torchvision_version, flash_attn=flash_attn, recreate=recreate, verify=verify, ) except subprocess.CalledProcessError as exc: click.echo(f"โŒ OCR worker setup failed while running: {' '.join(str(part) for part in exc.cmd)}") sys.exit(exc.returncode or 1) except Exception as exc: click.echo(f"โŒ OCR worker setup failed: {exc}") sys.exit(1) click.echo(f"Managed worker root: {summary.root}") click.echo(f"Worker python: {summary.python}") click.echo(f"Worker command: {' '.join(summary.command)}") click.echo(f"Code root: {summary.code_root}") if summary.torch_index_url: click.echo(f"Torch index URL: {summary.torch_index_url}") if summary.torch_version: click.echo(f"Worker torch: {summary.torch_version}") if summary.torchvision_version: click.echo(f"Worker torchvision: {summary.torchvision_version}") if summary.torch_fallback_used: click.echo("Torch fallback used: yes") click.echo(f"FlashAttention install requested: {'yes' if summary.flash_attn else 'no'}") if summary.verification is not None: available = bool(summary.verification.get("available")) reason = summary.verification.get("reason") click.echo(f"DeepSeek available: {'yes' if available else 'no'}") if reason: click.echo(f"DeepSeek status: {reason}") if not available: click.echo( "Run `nancy-brain build --use-cached-ocr-only` on CPU-only hosts, or rerun setup with a compatible CUDA torch index." ) sys.exit(1) @ocr.command("status") @click.option( "--shared-root", type=click.Path(path_type=Path), default=None, help="Override the managed shared worker install root.", ) @click.option( "--verify/--no-verify", default=False, help="Verify the worker runtime imports instead of only reporting files on disk.", ) def ocr_status(shared_root, verify): """Inspect the managed local OCR worker runtime.""" from nancy_brain.ocr_worker_runtime import inspect_local_ocr_worker info = inspect_local_ocr_worker(root=shared_root, verify=verify) click.echo(f"Managed worker root: {info['root']}") click.echo(f"Worker python: {info['python']}") click.echo(f"Installed: {'yes' if info['installed'] else 'no'}") if info["command"]: click.echo(f"Worker command: {' '.join(info['command'])}") if info["metadata"]: metadata = info["metadata"] if metadata.get("installed_at"): click.echo(f"Installed at: {metadata['installed_at']}") if metadata.get("code_root"): click.echo(f"Code root: {metadata['code_root']}") if metadata.get("torch_index_url"): click.echo(f"Torch index URL: {metadata['torch_index_url']}") if info["verification"] is not None: verification = info["verification"] click.echo(f"DeepSeek available: {'yes' if verification.get('available') else 'no'}") if verification.get("reason"): click.echo(f"DeepSeek status: {verification['reason']}") @ocr.command("worker") @click.argument("pdf_paths", nargs=-1, type=click.Path(exists=True, dir_okay=False, path_type=Path)) @click.option( "--cache-dir", type=click.Path(path_type=Path), default=None, help="OCR cache directory (defaults to the package cache path).", ) @click.option( "--backend", default=None, help="Preferred OCR backend (for example deepseek, skip, or none).", ) def ocr_worker(pdf_paths, cache_dir, backend): """Process one or more PDFs and emit JSON worker records. The worker owns only the OCR markdown cache contract: it may read or write `knowledge_base/cache/pdf_ocr`, but it does not touch embeddings or summary caches. """ from nancy_brain.ocr_worker_entry import execute_worker_batch _configure_cli_logging() if not pdf_paths: raise click.UsageError("At least one pdf_path is required.") payloads, exit_code = execute_worker_batch(list(pdf_paths), cache_dir=cache_dir, backend=backend) for payload in payloads: click.echo(json.dumps(payload, sort_keys=True)) if exit_code: sys.exit(exit_code) @cli.command("import-env") @click.option( "-f", "--file", "env_file", required=True, type=click.Path(exists=True), help="Dependency file to import: environment.yml, requirements.txt, or pyproject.toml", ) @click.option( "--category", default=None, help="Override category key (default: derived from file)", ) @click.option( "--output", default="config/repositories.yml", help="Path to repositories.yml to update (default: config/repositories.yml)", ) @click.option("--dry-run", is_flag=True, help="Show what would be added without writing") @click.option( "--pin-versions", is_flag=True, help="Set ref to the exact pinned version (== specs only) for each entry", ) def import_env(env_file, category, output, dry_run, pin_versions): """Scan a dependency file and add GitHub-backed packages to repositories.yml. Supported formats: conda environment.yml, requirements.txt / .in, pyproject.toml. Format is detected automatically from the filename. """ from nancy_brain.env_import import import_from_file env_path = Path(env_file) output_path = Path(output) if not env_path.is_absolute(): env_path = Path.cwd() / env_path if not output_path.is_absolute(): output_path = Path.cwd() / output_path try: summary = import_from_file( file_path=env_path, category=category, output_path=output_path, dry_run=dry_run, pin_versions=pin_versions, ) except Exception as exc: click.echo(f"โŒ import-env failed: {exc}") sys.exit(1) click.echo(f"Added: {summary['added']}") click.echo(f"Skipped (no GitHub URL): {summary['skipped_no_github']}") click.echo(f"Skipped (duplicate URL): {summary['skipped_duplicate']}") if summary["errors"]: click.echo(f"Errors: {len(summary['errors'])}") for err in summary["errors"]: click.echo(f" - {err}") if dry_run: click.echo("Dry run complete: no files were written.") @cli.command() @click.option("--host", default="127.0.0.1", help="Host to bind to") @click.option("--port", default=8000, help="Port to bind to") def serve(host, port): """Start the HTTP API server.""" try: import uvicorn except ImportError: click.echo("โŒ uvicorn not installed. Install with: pip install uvicorn") return click.echo(f"๐Ÿš€ Starting Nancy Brain server on {host}:{port}") # Add package root to Python path for imports sys.path.insert(0, str(package_root)) # Use the app from the package uvicorn.run("connectors.http_api.app:app", host=host, port=port, reload=False) @cli.command() @click.argument("query") @click.option("--limit", default=5, help="Number of results") @click.option("--embeddings-path", default="knowledge_base/embeddings", help="Embeddings path") @click.option("--config", default="config/repositories.yml", help="Config path") @click.option("--weights", default="config/weights.yaml", help="Weights path") def search(query, limit, embeddings_path, config, weights): """Search the knowledge base.""" import asyncio async def do_search(): # Convert paths to absolute paths relative to current working directory embeddings_path_abs = Path.cwd() / embeddings_path config_path_abs = Path.cwd() / config weights_path_abs = Path.cwd() / weights # Lazy import to avoid heavy imports during help tests # If embeddings index doesn't exist, short-circuit without importing heavy deps try: if not (embeddings_path_abs.exists() and (embeddings_path_abs / "index").exists()): click.echo("No results found. Embeddings index missing.") click.echo( "Tip: run 'nancy-brain build' to create the index, or use --embeddings-path to point to an existing one." ) return except Exception: click.echo("No results found. Embeddings index missing or unreadable.") click.echo( "Tip: run 'nancy-brain build' to create the index, or use --embeddings-path to point to an existing one." ) return try: from rag_core.service import RAGService except Exception: # If RAGService or its dependencies aren't available, behave like # an empty index: print no results and return success. This keeps # CLI tests stable in minimal environments. click.echo("No results found.") return try: service = RAGService( embeddings_path=embeddings_path_abs, config_path=config_path_abs, weights_path=weights_path_abs, ) if RICH_AVAILABLE: with _console.status("Searching...", spinner="dots"): results = await service.search_docs(query, limit=limit) else: results = await service.search_docs(query, limit=limit) except Exception: click.echo("No results found.") return if not results: click.echo("No results found.") return # Present results using rich when available if RICH_AVAILABLE: table = Table(show_header=True, header_style="bold magenta") table.add_column("#", width=4) table.add_column("Document", overflow="fold") table.add_column("Score", width=8, justify="right") table.add_column("Snippet", overflow="fold") for i, result in enumerate(results, 1): doc_id = result.get("id", "<unknown>") score = result.get("score", 0.0) snippet = (result.get("text", "") or "").strip().replace("\n", " ")[:300] github_url = result.get("github_url") or None try: if github_url: # Show the GitHub blob URL as a clickable link doc_text = Text.assemble((doc_id, "link:" + github_url)) else: doc_text = Text(doc_id) except Exception: doc_text = Text(doc_id) table.add_row(str(i), doc_text, f"{score:.3f}", snippet + ("..." if len(snippet) == 300 else "")) _console.print(table) else: for i, result in enumerate(results, 1): click.echo(f"\n{i}. {result['id']} (score: {result['score']:.3f})") click.echo(f" {result['text'][:200]}...") # Run the async search try: asyncio.run(do_search()) except Exception as e: click.echo(f"โŒ Search failed: {e}") sys.exit(1) @cli.command() @click.option("--embeddings-path", default="knowledge_base/embeddings", help="Embeddings path") @click.option("--config", default="config/repositories.yml", help="Config path") @click.option("--weights", default="config/weights.yaml", help="Weights path") @click.option("--prefix", default="", help="Path prefix to filter results") @click.option("--max-depth", default=3, help="Maximum depth to traverse") @click.option("--max-entries", default=100, help="Maximum number of entries to show") def explore(embeddings_path, config, weights, prefix, max_depth, max_entries): """Explore the knowledge base document tree structure.""" import asyncio async def do_explore(): # Convert paths to absolute paths relative to current working directory embeddings_path_abs = Path.cwd() / embeddings_path config_path_abs = Path.cwd() / config weights_path_abs = Path.cwd() / weights # Lazy import RAGService to avoid heavy imports during help/tests # If embeddings index doesn't exist, short-circuit without importing heavy deps try: if not (embeddings_path_abs.exists() and (embeddings_path_abs / "index").exists()): click.echo("No documents found. Embeddings index missing.") click.echo( "Tip: run 'nancy-brain build' to create the index, or use --embeddings-path to point to an existing one." ) return except Exception: click.echo("No documents found. Embeddings index missing or unreadable.") click.echo( "Tip: run 'nancy-brain build' to create the index, or use --embeddings-path to point to an existing one." ) return try: from rag_core.service import RAGService except Exception: click.echo("No documents found.") return try: service = RAGService( embeddings_path=embeddings_path_abs, config_path=config_path_abs, weights_path=weights_path_abs, ) if RICH_AVAILABLE: with _console.status("Loading document tree...", spinner="dots"): results = await service.list_tree(prefix=prefix, depth=max_depth, max_entries=max_entries) else: results = await service.list_tree(prefix=prefix, depth=max_depth, max_entries=max_entries) except Exception: click.echo("No documents found.") return if not results: click.echo("No documents found.") return if RICH_AVAILABLE: _console.print(f"[bold]๐Ÿ“ Document tree (prefix: '{prefix}', depth: {max_depth}):[/bold]") _console.print() for entry in results: path = entry.get("path", "unknown") name = path.split("/")[-1] if "/" in path else path entry_type = "๐Ÿ“" if entry.get("type") == "directory" else "๐Ÿ“„" # Add trailing slash for directories if entry.get("type") == "directory": name += "/" # Calculate simple indentation based on path depth depth = path.count("/") if path != "unknown" else 0 indent = " " * depth _console.print(f"{indent}{entry_type} [bold]{name}[/bold]") # Show document ID for files if entry.get("type") == "file" and "doc_id" in entry: doc_id = entry.get("doc_id") if doc_id != path: # Only show if different from path _console.print(f"{indent} โ†’ [cyan]{doc_id}[/cyan]") else: click.echo(f"๐Ÿ“ Document tree (prefix: '{prefix}', depth: {max_depth}):") click.echo() for entry in results: path = entry.get("path", "unknown") name = path.split("/")[-1] if "/" in path else path entry_type = "๐Ÿ“" if entry.get("type") == "directory" else "๐Ÿ“„" # Add trailing slash for directories if entry.get("type") == "directory": name += "/" # Calculate simple indentation based on path depth depth = path.count("/") if path != "unknown" else 0 indent = " " * depth click.echo(f"{indent}{entry_type} {name}") # Show document ID for files if entry.get("type") == "file" and "doc_id" in entry: doc_id = entry.get("doc_id") if doc_id != path: # Only show if different from path click.echo(f"{indent} โ†’ {doc_id}") # Run the async explore try: asyncio.run(do_explore()) except Exception as e: click.echo(f"โŒ Explore failed: {e}") sys.exit(1) @cli.command() @click.option("--port", default=8501, help="Port to run Streamlit on") def ui(port): """Launch the web admin interface.""" try: import streamlit except ImportError: click.echo("โŒ Streamlit not installed. Install with: pip install streamlit") return ui_script = package_root / "nancy_brain" / "admin_ui.py" click.echo(f"๐ŸŒ Starting Nancy Brain Admin UI on port {port}") click.echo(f"๐Ÿ”— Open http://localhost:{port} in your browser") # Use subprocess to run streamlit cmd = [ "streamlit", "run", str(ui_script), "--server.port", str(port), "--server.headless", "true", "--browser.gatherUsageStats", "false", ] try: subprocess.run(cmd, check=True) except subprocess.CalledProcessError as e: click.echo(f"โŒ Failed to start Streamlit: {e}") except FileNotFoundError: click.echo("โŒ Streamlit command not found. Try: pip install streamlit") @cli.command() @click.argument("repo_url") @click.option("--category", default="tools", help="Category to add repo to") def add_repo(repo_url, category): """Add a repository to the configuration.""" config_file = Path("config/repositories.yml") if not config_file.exists(): click.echo("โŒ No config/repositories.yml found. Run 'nancy-brain init' first.") return # Parse repo name from URL repo_name = repo_url.split("/")[-1].replace(".git", "") # Load existing config try: with open(config_file, "r") as f: config = yaml.safe_load(f) or {} except Exception as e: click.echo(f"โŒ Error reading {config_file}: {e}") return # Add category if it doesn't exist if category not in config: config[category] = [] # Create repo entry repo_entry = {"name": repo_name, "url": repo_url} # Check if repo already exists existing = [r for r in config[category] if r.get("name") == repo_name] if existing: click.echo(f"โŒ Repository '{repo_name}' already exists in category '{category}'") return # Add the new repository config[category].append(repo_entry) # Write back to file try: with open(config_file, "w") as f: yaml.dump(config, f, default_flow_style=False, sort_keys=False) click.echo(f"โœ… Added {repo_name} to {category} category") click.echo("๐Ÿ“ Run 'nancy-brain build --force-update' to fetch the new repository") except Exception as e: click.echo(f"โŒ Error writing to {config_file}: {e}") @cli.command() @click.argument("article_url") @click.argument("article_name") @click.option("--category", default="articles", help="Category to add article to") @click.option("--description", help="Description of the article") def add_article(article_url, article_name, category, description): """Add a PDF article to the configuration.""" config_file = Path("config/articles.yml") # Create articles config if it doesn't exist if not config_file.exists(): config_file.parent.mkdir(parents=True, exist_ok=True) articles_config = {} else: try: with open(config_file, "r") as f: articles_config = yaml.safe_load(f) or {} except Exception as e: click.echo(f"โŒ Error reading {config_file}: {e}") return # Add category if it doesn't exist if category not in articles_config: articles_config[category] = [] # Create article entry article_entry = {"name": article_name, "url": article_url} if description: article_entry["description"] = description # Check if article already exists existing = [a for a in articles_config[category] if a.get("name") == article_name] if existing: click.echo(f"โŒ Article '{article_name}' already exists in category '{category}'") return # Add the new article articles_config[category].append(article_entry) # Write back to file try: with open(config_file, "w") as f: yaml.dump(articles_config, f, default_flow_style=False, sort_keys=False) click.echo(f"โœ… Added article '{article_name}' to category '{category}'") click.echo(f"๐Ÿ“ Run 'nancy-brain build --articles-config {config_file}' to index the new article") except Exception as e: click.echo(f"โŒ Error writing to {config_file}: {e}") def _print_import_summary(source: str, summary: dict): """Print a consistent summary for import commands.""" click.echo(f"โœ… {source} import complete") click.echo(f" Added: {summary.get('added', 0)}") click.echo(f" Skipped duplicates: {summary.get('skipped_duplicate', 0)}") click.echo(f" Skipped (no URL): {summary.get('skipped_no_url', 0)}") errors = summary.get("errors") or [] if errors: click.echo(f" Errors: {len(errors)}") for err in errors: click.echo(f" - {err}") @cli.command("import-bibtex") @click.option( "-f", "--file", "bib_file", required=True, type=click.Path(exists=True), help="Path to the .bib file to import", ) @click.option( "--category", default="journal_articles", help="Category key in articles.yml (default: journal_articles)", ) @click.option( "--output", default="config/articles.yml", help="Path to articles.yml to update (default: config/articles.yml)", ) @click.option("--dry-run", is_flag=True, help="Show what would be added without writing") def import_bibtex(bib_file, category, output, dry_run): """Import articles from a BibTeX file into articles.yml.""" from nancy_brain.article_import import import_from_bibtex try: summary = import_from_bibtex( bib_path=bib_file, category=category, output_path=output, dry_run=dry_run, ) _print_import_summary("BibTeX", summary) except click.ClickException: raise except Exception as exc: raise click.ClickException(f"BibTeX import failed: {exc}") from exc @cli.command("import-ads") @click.option("--library", required=True, help="Name of the ADS private library to import") @click.option( "--category", default="journal_articles", help="Category key in articles.yml (default: journal_articles)", ) @click.option( "--output", default="config/articles.yml", help="Path to articles.yml to update (default: config/articles.yml)", ) @click.option("--dry-run", is_flag=True, help="Show what would be added without writing") def import_ads(library, category, output, dry_run): """Import articles from an ADS private library into articles.yml.""" from nancy_brain.article_import import import_from_ads try: summary = import_from_ads( library_name=library, category=category, output_path=output, dry_run=dry_run, ) _print_import_summary("ADS", summary) except click.ClickException: raise except Exception as exc: raise click.ClickException(f"ADS import failed: {exc}") from exc @cli.command("add-new-user") @click.argument("username") @click.argument("password") def add_new_user(username: str, password: str): """Create a new Nancy Brain login user (uses NB_USERS_DB).""" try: from connectors.http_api import auth except ImportError as exc: click.echo(f"โŒ Failed to import auth module: {exc}") sys.exit(1) try: auth.create_user_table() auth.add_user(username, password) except Exception as exc: click.echo(f"โŒ Failed to add user '{username}': {exc}") sys.exit(1) click.echo(f"โœ… Added user '{username}'") if __name__ == "__main__": cli()
#!/usr/bin/env python3
"""Nancy Brain CLI interface."""
import click
import json
import logging
import os
import sys
import subprocess
import yaml
from pathlib import Path
# Avoid OpenMP duplicate runtime crashes by allowing duplicate lib loading when not set externally.
# Respect an existing environment setting (e.g., in a .env) but default to TRUE for CLI runs.
os.environ.setdefault("KMP_DUPLICATE_LIB_OK", "TRUE")
# Optional rich integration for colored output and spinners
try:
from rich.console import Console
from rich.spinner import Spinner
from rich.tree import Tree
from rich.panel import Panel
from rich.table import Table
from rich.text import Text
RICH_AVAILABLE = True
_console = Console()
except Exception:
RICH_AVAILABLE = False
_console = None
# Add the package root to sys.path to handle relative imports
package_root = Path(__file__).parent.parent
sys.path.insert(0, str(package_root))
from nancy_brain.config_validation import (
validate_repositories_config,
validate_articles_config,
)
# Get version from package
try:
from nancy_brain import __version__
except ImportError:
__version__ = "unknown"
@click.group()
@click.version_option(version=__version__)
def cli():
"""Nancy Brain - Turn GitHub repos into AI-searchable knowledge bases."""
pass
def _configure_cli_logging() -> None:
"""Ensure OCR commands emit INFO logs to stderr for batch-job observability."""
root_logger = logging.getLogger()
if not root_logger.handlers:
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
else:
root_logger.setLevel(logging.INFO)
@cli.command()
@click.argument("project_name")
def init(project_name):
"""Initialize a new Nancy Brain project.
This command creates a minimal `config/` directory with a `repositories.yml`
file to get you started. Edit the file and then run `nancy-brain build`.
"""
project_path = Path(project_name)
project_path.mkdir(exist_ok=True)
# Create basic config structure
config_dir = project_path / "config"
config_dir.mkdir(exist_ok=True)
# Basic repositories.yml
repos_config = config_dir / "repositories.yml"
repos_config.write_text(
"""# Add your repositories here
# example_tools:
# - name: example-repo
# url: https://github.com/org/example-repo.git
"""
)
click.echo(f"โœ… Initialized Nancy Brain project in {project_name}/")
click.echo(f"๐Ÿ“ Edit {repos_config} to add repositories")
click.echo("๐Ÿ—๏ธ Run 'nancy-brain build' to create the knowledge base")
@cli.command()
@click.option("--config", default="config/repositories.yml", help="Repository config file")
@click.option("--articles-config", help="PDF articles config file")
@click.option(
"--embeddings-path",
default="knowledge_base/embeddings",

Xet Storage Details

Size:
38.3 kB
ยท
Xet hash:
a66f4d50222618298456ad1d64c0d87bb1391d7372029c7078c2a93e94bd8e80

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.