Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /AmberLee2427_nancy-brain /nancy_brain /cli.py
#!/usr/bin/env python3
"""Nancy Brain CLI interface."""
import click
import json
import logging
import os
import sys
import subprocess
import yaml
from pathlib import Path
# Avoid OpenMP duplicate runtime crashes by allowing duplicate lib loading when not set externally.
# Respect an existing environment setting (e.g., in a .env) but default to TRUE for CLI runs.
os.environ.setdefault("KMP_DUPLICATE_LIB_OK", "TRUE")
# Optional rich integration for colored output and spinners
try:
from rich.console import Console
from rich.spinner import Spinner
from rich.tree import Tree
from rich.panel import Panel
from rich.table import Table
from rich.text import Text
RICH_AVAILABLE = True
_console = Console()
except Exception:
RICH_AVAILABLE = False
_console = None
# Add the package root to sys.path to handle relative imports
package_root = Path(__file__).parent.parent
sys.path.insert(0, str(package_root))
from nancy_brain.config_validation import (
validate_repositories_config,
validate_articles_config,
)
# Get version from package
try:
from nancy_brain import __version__
except ImportError:
__version__ = "unknown"
@click.group()
@click.version_option(version=__version__)
def cli():
"""Nancy Brain - Turn GitHub repos into AI-searchable knowledge bases."""
pass
def _configure_cli_logging() -> None:
"""Ensure OCR commands emit INFO logs to stderr for batch-job observability."""
root_logger = logging.getLogger()
if not root_logger.handlers:
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
else:
root_logger.setLevel(logging.INFO)
@cli.command()
@click.argument("project_name")
def init(project_name):
"""Initialize a new Nancy Brain project.
This command creates a minimal `config/` directory with a `repositories.yml`
file to get you started. Edit the file and then run `nancy-brain build`.
"""
project_path = Path(project_name)
project_path.mkdir(exist_ok=True)
# Create basic config structure
config_dir = project_path / "config"
config_dir.mkdir(exist_ok=True)
# Basic repositories.yml
repos_config = config_dir / "repositories.yml"
repos_config.write_text(
"""# Add your repositories here
# example_tools:
# - name: example-repo
# url: https://github.com/org/example-repo.git
"""
)
click.echo(f"โ
Initialized Nancy Brain project in {project_name}/")
click.echo(f"๐ Edit {repos_config} to add repositories")
click.echo("๐๏ธ Run 'nancy-brain build' to create the knowledge base")
@cli.command()
@click.option("--config", default="config/repositories.yml", help="Repository config file")
@click.option("--articles-config", help="PDF articles config file")
@click.option(
"--embeddings-path",
default="knowledge_base/embeddings",
help="Embeddings output path",
)
@click.option("--force-update", is_flag=True, help="Force update all repositories")
@click.option("--dry-run", is_flag=True, help="Show what would be done without executing the build")
@click.option("--dirty", is_flag=True, help="Leave raw repos and PDFs in place after build (don't cleanup)")
@click.option(
"--summaries/--no-summaries",
default=None,
help="Generate document summaries during build (local, custom OpenAI-compatible, or Anthropic)",
)
@click.option(
"--batch-size",
default=0,
type=int,
help="Index documents in batches (requires embeddings.upsert support). 0 = disable batching.",
)
@click.option(
"--max-docs",
default=0,
type=int,
help="Stop after indexing this many document chunks (for testing / limiting resource use). 0 = no limit.",
)
@click.option(
"--category",
help="Limit build to a single repository category (as defined in repositories.yml)",
)
@click.option(
"--repo",
default=None,
help="Limit build to a single repository by name (across all categories unless --category is set).",
)
@click.option(
"--summaries-only",
is_flag=True,
default=False,
help="Warm the summary cache per-repo then exit without building the index. Pair with --repo for one-node-per-repo cluster jobs.",
)
@click.option(
"--use-cached-ocr-only",
is_flag=True,
default=False,
help="Only use cached OCR Markdown for PDFs. Missing cache entries are deferred as needs_ocr.",
)
def build(
config,
articles_config,
embeddings_path,
force_update,
dry_run,
dirty,
summaries,
batch_size,
max_docs,
category,
repo,
summaries_only,
use_cached_ocr_only,
):
"""Build the knowledge base from configured repositories.
The build command validates `config/repositories.yml` (and `config/articles.yml`
if provided) before starting. If validation fails, the command prints
detailed errors and exits with a non-zero status.
"""
click.echo("๐๏ธ Building knowledge base...")
# Convert paths to absolute paths relative to current working directory
config_path = Path.cwd() / config
embeddings_path = Path.cwd() / embeddings_path
# Pre-validate repository config to provide immediate feedback
if not config_path.exists():
click.echo(f"โ Repository config not found: {config_path}")
sys.exit(2)
try:
with open(config_path, "r", encoding="utf-8") as f:
cfg = yaml.safe_load(f) or {}
except Exception as e:
click.echo(f"โ Failed to read repository config: {e}")
sys.exit(2)
ok, errors = validate_repositories_config(cfg)
if not ok:
click.echo("โ repositories.yml validation failed:")
for err in errors:
click.echo(f" - {err}")
sys.exit(2)
# Build command arguments
cmd = [
sys.executable,
str(package_root / "scripts" / "build_knowledge_base.py"),
"--config",
str(config_path),
"--embeddings-path",
str(embeddings_path),
]
if articles_config:
articles_config_path = Path.cwd() / articles_config
# Validate articles config as well
if not articles_config_path.exists():
click.echo(f"โ Articles config not found: {articles_config_path}")
sys.exit(2)
try:
with open(articles_config_path, "r", encoding="utf-8") as f:
a_cfg = yaml.safe_load(f) or {}
except Exception as e:
click.echo(f"โ Failed to read articles config: {e}")
sys.exit(2)
ok2, errs2 = validate_articles_config(a_cfg)
if not ok2:
click.echo("โ articles.yml validation failed:")
for err in errs2:
click.echo(f" - {err}")
sys.exit(2)
cmd.extend(["--articles-config", str(articles_config_path)])
if force_update:
cmd.append("--force-update")
if dirty:
cmd.append("--dirty")
if summaries is True:
cmd.append("--summaries")
elif summaries is False:
cmd.append("--no-summaries")
if batch_size and batch_size > 0:
cmd.extend(["--batch-size", str(batch_size)])
if max_docs and max_docs > 0:
cmd.extend(["--max-docs", str(max_docs)])
if category:
cmd.extend(["--category", category])
if repo:
cmd.extend(["--repo", repo])
if summaries_only:
cmd.append("--summaries-only")
if use_cached_ocr_only:
cmd.append("--use-cached-ocr-only")
# If dry-run requested, still run the underlying script with --dry-run so that
# repository cloning/downloading/indexing intentions and validation summaries
# are produced by the central pipeline logic rather than a hollow preview.
if dry_run:
cmd.append("--dry-run")
if RICH_AVAILABLE:
_console.print("[yellow]๐ Dry run: executing pipeline in no-op mode[/yellow]")
else:
click.echo(click.style("๐ Dry run: executing pipeline in no-op mode", fg="yellow"))
# Run the build script from the package directory
try:
if RICH_AVAILABLE:
with _console.status("Building knowledge base...", spinner="dots"):
result = subprocess.run(cmd, check=True)
else:
result = subprocess.run(cmd, check=True)
success_msg = "โ
Knowledge base built successfully!"
if RICH_AVAILABLE:
_console.print(f"[green]{success_msg}[/green]")
else:
click.echo(click.style(success_msg, fg="green"))
except subprocess.CalledProcessError as e:
err_msg = f"โ Build failed with exit code {e.returncode}"
if RICH_AVAILABLE:
_console.print(f"[red]{err_msg}[/red]")
else:
click.echo(click.style(err_msg, fg="red"))
sys.exit(e.returncode)
@cli.group()
def ocr():
"""OCR utilities and cache warmers."""
pass
@ocr.command("warm")
@click.argument("paths", nargs=-1, type=click.Path(path_type=Path))
@click.option(
"--cache-dir",
type=click.Path(path_type=Path),
default=None,
help="OCR cache directory (defaults to the package cache path).",
)
@click.option(
"--backend",
default=None,
help="Preferred OCR backend (for example deepseek, skip, or none).",
)
@click.option(
"--articles-config",
type=click.Path(path_type=Path),
default=None,
help="Optional articles.yml to download before warming OCR cache.",
)
@click.option(
"--base-path",
type=click.Path(path_type=Path),
default=Path("knowledge_base/raw"),
show_default=True,
help="Base path for downloaded PDFs and scan roots.",
)
@click.option(
"--recursive/--no-recursive",
default=True,
help="Recurse into directories when scanning for PDFs.",
)
def ocr_warm(paths, cache_dir, backend, articles_config, base_path, recursive):
"""Warm OCR cache entries for PDFs."""
from nancy_brain.pdf_ocr import warm_pdf_ocr_cache
_configure_cli_logging()
scan_paths = list(paths)
if articles_config:
from scripts.build_knowledge_base import download_pdf_articles
download_pdf_articles(str(articles_config), base_path=str(base_path))
article_paths = _existing_article_pdf_paths(articles_config, base_path)
scan_paths.extend(article_paths)
if not scan_paths:
if articles_config:
click.echo(f"No article PDFs found under {base_path} for config {articles_config}")
sys.exit(1)
scan_paths = [base_path]
results = warm_pdf_ocr_cache(scan_paths, cache_dir=cache_dir, preferred_backend=backend, recursive=recursive)
generated = sum(1 for result in results if result.status == "generated")
cached = sum(1 for result in results if result.cached)
deferred = sum(1 for result in results if result.needs_ocr)
failed = sum(1 for result in results if result.status == "error")
click.echo(f"PDFs scanned: {len(results)}")
click.echo(f"Cached hits: {cached}")
click.echo(f"Generated: {generated}")
click.echo(f"Deferred (needs OCR): {deferred}")
click.echo(f"Failed: {failed}")
for result in results:
if result.needs_ocr and result.warning:
click.echo(f" - needs_ocr: {result.warning}")
elif result.status == "error" and result.warning:
click.echo(f" - error: {result.warning}")
if failed or ((generated + cached) == 0 and deferred > 0):
if failed == 0:
click.echo("No OCR cache entries were generated or reused; all PDFs were deferred for OCR.")
sys.exit(1)
def _existing_article_pdf_paths(articles_config: Path, base_path: Path) -> list[Path]:
"""Return existing article PDFs declared in the provided articles config."""
with open(articles_config, "r", encoding="utf-8") as handle:
config = yaml.safe_load(handle) or {}
seen: set[Path] = set()
article_paths: list[Path] = []
for category, articles in config.items():
if not isinstance(articles, list):
continue
for article in articles:
if not isinstance(article, dict):
continue
article_name = str(article.get("name", "")).strip()
if not article_name:
continue
pdf_path = (base_path / category / f"{article_name}.pdf").resolve(strict=False)
if pdf_path.exists() and pdf_path not in seen:
article_paths.append(pdf_path)
seen.add(pdf_path)
return article_paths
@ocr.command("setup")
@click.option(
"--shared-root",
type=click.Path(path_type=Path),
default=None,
help="Override the managed shared worker install root.",
)
@click.option(
"--python",
"python_executable",
type=click.Path(path_type=Path),
default=None,
help="Python executable used to create the worker venv (defaults to the current interpreter).",
)
@click.option(
"--torch-index-url",
default=None,
help="Optional PyTorch wheel index URL for the isolated worker runtime.",
)
@click.option(
"--torch-version",
default=None,
help="Optional torch version override for the worker runtime.",
)
@click.option(
"--torchvision-version",
default=None,
help="Optional torchvision version override for the worker runtime.",
)
@click.option(
"--flash-attn/--no-flash-attn",
default=False,
help="Attempt to install flash-attn into the worker runtime.",
)
@click.option(
"--recreate",
is_flag=True,
default=False,
help="Recreate the worker virtualenv from scratch before installing packages.",
)
@click.option(
"--verify/--no-verify",
default=True,
help="Verify the installed worker runtime after setup.",
)
def ocr_setup(
shared_root,
python_executable,
torch_index_url,
torch_version,
torchvision_version,
flash_attn,
recreate,
verify,
):
"""Install or update the managed local OCR worker runtime."""
from nancy_brain.ocr_worker_runtime import install_local_ocr_worker
try:
summary = install_local_ocr_worker(
root=shared_root,
python_executable=python_executable,
torch_index_url=torch_index_url,
torch_version=torch_version,
torchvision_version=torchvision_version,
flash_attn=flash_attn,
recreate=recreate,
verify=verify,
)
except subprocess.CalledProcessError as exc:
click.echo(f"โ OCR worker setup failed while running: {' '.join(str(part) for part in exc.cmd)}")
sys.exit(exc.returncode or 1)
except Exception as exc:
click.echo(f"โ OCR worker setup failed: {exc}")
sys.exit(1)
click.echo(f"Managed worker root: {summary.root}")
click.echo(f"Worker python: {summary.python}")
click.echo(f"Worker command: {' '.join(summary.command)}")
click.echo(f"Code root: {summary.code_root}")
if summary.torch_index_url:
click.echo(f"Torch index URL: {summary.torch_index_url}")
if summary.torch_version:
click.echo(f"Worker torch: {summary.torch_version}")
if summary.torchvision_version:
click.echo(f"Worker torchvision: {summary.torchvision_version}")
if summary.torch_fallback_used:
click.echo("Torch fallback used: yes")
click.echo(f"FlashAttention install requested: {'yes' if summary.flash_attn else 'no'}")
if summary.verification is not None:
available = bool(summary.verification.get("available"))
reason = summary.verification.get("reason")
click.echo(f"DeepSeek available: {'yes' if available else 'no'}")
if reason:
click.echo(f"DeepSeek status: {reason}")
if not available:
click.echo(
"Run `nancy-brain build --use-cached-ocr-only` on CPU-only hosts, or rerun setup with a compatible CUDA torch index."
)
sys.exit(1)
@ocr.command("status")
@click.option(
"--shared-root",
type=click.Path(path_type=Path),
default=None,
help="Override the managed shared worker install root.",
)
@click.option(
"--verify/--no-verify",
default=False,
help="Verify the worker runtime imports instead of only reporting files on disk.",
)
def ocr_status(shared_root, verify):
"""Inspect the managed local OCR worker runtime."""
from nancy_brain.ocr_worker_runtime import inspect_local_ocr_worker
info = inspect_local_ocr_worker(root=shared_root, verify=verify)
click.echo(f"Managed worker root: {info['root']}")
click.echo(f"Worker python: {info['python']}")
click.echo(f"Installed: {'yes' if info['installed'] else 'no'}")
if info["command"]:
click.echo(f"Worker command: {' '.join(info['command'])}")
if info["metadata"]:
metadata = info["metadata"]
if metadata.get("installed_at"):
click.echo(f"Installed at: {metadata['installed_at']}")
if metadata.get("code_root"):
click.echo(f"Code root: {metadata['code_root']}")
if metadata.get("torch_index_url"):
click.echo(f"Torch index URL: {metadata['torch_index_url']}")
if info["verification"] is not None:
verification = info["verification"]
click.echo(f"DeepSeek available: {'yes' if verification.get('available') else 'no'}")
if verification.get("reason"):
click.echo(f"DeepSeek status: {verification['reason']}")
@ocr.command("worker")
@click.argument("pdf_paths", nargs=-1, type=click.Path(exists=True, dir_okay=False, path_type=Path))
@click.option(
"--cache-dir",
type=click.Path(path_type=Path),
default=None,
help="OCR cache directory (defaults to the package cache path).",
)
@click.option(
"--backend",
default=None,
help="Preferred OCR backend (for example deepseek, skip, or none).",
)
def ocr_worker(pdf_paths, cache_dir, backend):
"""Process one or more PDFs and emit JSON worker records.
The worker owns only the OCR markdown cache contract:
it may read or write `knowledge_base/cache/pdf_ocr`, but it does not
touch embeddings or summary caches.
"""
from nancy_brain.ocr_worker_entry import execute_worker_batch
_configure_cli_logging()
if not pdf_paths:
raise click.UsageError("At least one pdf_path is required.")
payloads, exit_code = execute_worker_batch(list(pdf_paths), cache_dir=cache_dir, backend=backend)
for payload in payloads:
click.echo(json.dumps(payload, sort_keys=True))
if exit_code:
sys.exit(exit_code)
@cli.command("import-env")
@click.option(
"-f",
"--file",
"env_file",
required=True,
type=click.Path(exists=True),
help="Dependency file to import: environment.yml, requirements.txt, or pyproject.toml",
)
@click.option(
"--category",
default=None,
help="Override category key (default: derived from file)",
)
@click.option(
"--output",
default="config/repositories.yml",
help="Path to repositories.yml to update (default: config/repositories.yml)",
)
@click.option("--dry-run", is_flag=True, help="Show what would be added without writing")
@click.option(
"--pin-versions",
is_flag=True,
help="Set ref to the exact pinned version (== specs only) for each entry",
)
def import_env(env_file, category, output, dry_run, pin_versions):
"""Scan a dependency file and add GitHub-backed packages to repositories.yml.
Supported formats: conda environment.yml, requirements.txt / .in, pyproject.toml.
Format is detected automatically from the filename.
"""
from nancy_brain.env_import import import_from_file
env_path = Path(env_file)
output_path = Path(output)
if not env_path.is_absolute():
env_path = Path.cwd() / env_path
if not output_path.is_absolute():
output_path = Path.cwd() / output_path
try:
summary = import_from_file(
file_path=env_path,
category=category,
output_path=output_path,
dry_run=dry_run,
pin_versions=pin_versions,
)
except Exception as exc:
click.echo(f"โ import-env failed: {exc}")
sys.exit(1)
click.echo(f"Added: {summary['added']}")
click.echo(f"Skipped (no GitHub URL): {summary['skipped_no_github']}")
click.echo(f"Skipped (duplicate URL): {summary['skipped_duplicate']}")
if summary["errors"]:
click.echo(f"Errors: {len(summary['errors'])}")
for err in summary["errors"]:
click.echo(f" - {err}")
if dry_run:
click.echo("Dry run complete: no files were written.")
@cli.command()
@click.option("--host", default="127.0.0.1", help="Host to bind to")
@click.option("--port", default=8000, help="Port to bind to")
def serve(host, port):
"""Start the HTTP API server."""
try:
import uvicorn
except ImportError:
click.echo("โ uvicorn not installed. Install with: pip install uvicorn")
return
click.echo(f"๐ Starting Nancy Brain server on {host}:{port}")
# Add package root to Python path for imports
sys.path.insert(0, str(package_root))
# Use the app from the package
uvicorn.run("connectors.http_api.app:app", host=host, port=port, reload=False)
@cli.command()
@click.argument("query")
@click.option("--limit", default=5, help="Number of results")
@click.option("--embeddings-path", default="knowledge_base/embeddings", help="Embeddings path")
@click.option("--config", default="config/repositories.yml", help="Config path")
@click.option("--weights", default="config/weights.yaml", help="Weights path")
def search(query, limit, embeddings_path, config, weights):
"""Search the knowledge base."""
import asyncio
async def do_search():
# Convert paths to absolute paths relative to current working directory
embeddings_path_abs = Path.cwd() / embeddings_path
config_path_abs = Path.cwd() / config
weights_path_abs = Path.cwd() / weights
# Lazy import to avoid heavy imports during help tests
# If embeddings index doesn't exist, short-circuit without importing heavy deps
try:
if not (embeddings_path_abs.exists() and (embeddings_path_abs / "index").exists()):
click.echo("No results found. Embeddings index missing.")
click.echo(
"Tip: run 'nancy-brain build' to create the index, or use --embeddings-path to point to an existing one."
)
return
except Exception:
click.echo("No results found. Embeddings index missing or unreadable.")
click.echo(
"Tip: run 'nancy-brain build' to create the index, or use --embeddings-path to point to an existing one."
)
return
try:
from rag_core.service import RAGService
except Exception:
# If RAGService or its dependencies aren't available, behave like
# an empty index: print no results and return success. This keeps
# CLI tests stable in minimal environments.
click.echo("No results found.")
return
try:
service = RAGService(
embeddings_path=embeddings_path_abs,
config_path=config_path_abs,
weights_path=weights_path_abs,
)
if RICH_AVAILABLE:
with _console.status("Searching...", spinner="dots"):
results = await service.search_docs(query, limit=limit)
else:
results = await service.search_docs(query, limit=limit)
except Exception:
click.echo("No results found.")
return
if not results:
click.echo("No results found.")
return
# Present results using rich when available
if RICH_AVAILABLE:
table = Table(show_header=True, header_style="bold magenta")
table.add_column("#", width=4)
table.add_column("Document", overflow="fold")
table.add_column("Score", width=8, justify="right")
table.add_column("Snippet", overflow="fold")
for i, result in enumerate(results, 1):
doc_id = result.get("id", "<unknown>")
score = result.get("score", 0.0)
snippet = (result.get("text", "") or "").strip().replace("\n", " ")[:300]
github_url = result.get("github_url") or None
try:
if github_url:
# Show the GitHub blob URL as a clickable link
doc_text = Text.assemble((doc_id, "link:" + github_url))
else:
doc_text = Text(doc_id)
except Exception:
doc_text = Text(doc_id)
table.add_row(str(i), doc_text, f"{score:.3f}", snippet + ("..." if len(snippet) == 300 else ""))
_console.print(table)
else:
for i, result in enumerate(results, 1):
click.echo(f"\n{i}. {result['id']} (score: {result['score']:.3f})")
click.echo(f" {result['text'][:200]}...")
# Run the async search
try:
asyncio.run(do_search())
except Exception as e:
click.echo(f"โ Search failed: {e}")
sys.exit(1)
@cli.command()
@click.option("--embeddings-path", default="knowledge_base/embeddings", help="Embeddings path")
@click.option("--config", default="config/repositories.yml", help="Config path")
@click.option("--weights", default="config/weights.yaml", help="Weights path")
@click.option("--prefix", default="", help="Path prefix to filter results")
@click.option("--max-depth", default=3, help="Maximum depth to traverse")
@click.option("--max-entries", default=100, help="Maximum number of entries to show")
def explore(embeddings_path, config, weights, prefix, max_depth, max_entries):
"""Explore the knowledge base document tree structure."""
import asyncio
async def do_explore():
# Convert paths to absolute paths relative to current working directory
embeddings_path_abs = Path.cwd() / embeddings_path
config_path_abs = Path.cwd() / config
weights_path_abs = Path.cwd() / weights
# Lazy import RAGService to avoid heavy imports during help/tests
# If embeddings index doesn't exist, short-circuit without importing heavy deps
try:
if not (embeddings_path_abs.exists() and (embeddings_path_abs / "index").exists()):
click.echo("No documents found. Embeddings index missing.")
click.echo(
"Tip: run 'nancy-brain build' to create the index, or use --embeddings-path to point to an existing one."
)
return
except Exception:
click.echo("No documents found. Embeddings index missing or unreadable.")
click.echo(
"Tip: run 'nancy-brain build' to create the index, or use --embeddings-path to point to an existing one."
)
return
try:
from rag_core.service import RAGService
except Exception:
click.echo("No documents found.")
return
try:
service = RAGService(
embeddings_path=embeddings_path_abs,
config_path=config_path_abs,
weights_path=weights_path_abs,
)
if RICH_AVAILABLE:
with _console.status("Loading document tree...", spinner="dots"):
results = await service.list_tree(prefix=prefix, depth=max_depth, max_entries=max_entries)
else:
results = await service.list_tree(prefix=prefix, depth=max_depth, max_entries=max_entries)
except Exception:
click.echo("No documents found.")
return
if not results:
click.echo("No documents found.")
return
if RICH_AVAILABLE:
_console.print(f"[bold]๐ Document tree (prefix: '{prefix}', depth: {max_depth}):[/bold]")
_console.print()
for entry in results:
path = entry.get("path", "unknown")
name = path.split("/")[-1] if "/" in path else path
entry_type = "๐" if entry.get("type") == "directory" else "๐"
# Add trailing slash for directories
if entry.get("type") == "directory":
name += "/"
# Calculate simple indentation based on path depth
depth = path.count("/") if path != "unknown" else 0
indent = " " * depth
_console.print(f"{indent}{entry_type} [bold]{name}[/bold]")
# Show document ID for files
if entry.get("type") == "file" and "doc_id" in entry:
doc_id = entry.get("doc_id")
if doc_id != path: # Only show if different from path
_console.print(f"{indent} โ [cyan]{doc_id}[/cyan]")
else:
click.echo(f"๐ Document tree (prefix: '{prefix}', depth: {max_depth}):")
click.echo()
for entry in results:
path = entry.get("path", "unknown")
name = path.split("/")[-1] if "/" in path else path
entry_type = "๐" if entry.get("type") == "directory" else "๐"
# Add trailing slash for directories
if entry.get("type") == "directory":
name += "/"
# Calculate simple indentation based on path depth
depth = path.count("/") if path != "unknown" else 0
indent = " " * depth
click.echo(f"{indent}{entry_type} {name}")
# Show document ID for files
if entry.get("type") == "file" and "doc_id" in entry:
doc_id = entry.get("doc_id")
if doc_id != path: # Only show if different from path
click.echo(f"{indent} โ {doc_id}")
# Run the async explore
try:
asyncio.run(do_explore())
except Exception as e:
click.echo(f"โ Explore failed: {e}")
sys.exit(1)
@cli.command()
@click.option("--port", default=8501, help="Port to run Streamlit on")
def ui(port):
"""Launch the web admin interface."""
try:
import streamlit
except ImportError:
click.echo("โ Streamlit not installed. Install with: pip install streamlit")
return
ui_script = package_root / "nancy_brain" / "admin_ui.py"
click.echo(f"๐ Starting Nancy Brain Admin UI on port {port}")
click.echo(f"๐ Open http://localhost:{port} in your browser")
# Use subprocess to run streamlit
cmd = [
"streamlit",
"run",
str(ui_script),
"--server.port",
str(port),
"--server.headless",
"true",
"--browser.gatherUsageStats",
"false",
]
try:
subprocess.run(cmd, check=True)
except subprocess.CalledProcessError as e:
click.echo(f"โ Failed to start Streamlit: {e}")
except FileNotFoundError:
click.echo("โ Streamlit command not found. Try: pip install streamlit")
@cli.command()
@click.argument("repo_url")
@click.option("--category", default="tools", help="Category to add repo to")
def add_repo(repo_url, category):
"""Add a repository to the configuration."""
config_file = Path("config/repositories.yml")
if not config_file.exists():
click.echo("โ No config/repositories.yml found. Run 'nancy-brain init' first.")
return
# Parse repo name from URL
repo_name = repo_url.split("/")[-1].replace(".git", "")
# Load existing config
try:
with open(config_file, "r") as f:
config = yaml.safe_load(f) or {}
except Exception as e:
click.echo(f"โ Error reading {config_file}: {e}")
return
# Add category if it doesn't exist
if category not in config:
config[category] = []
# Create repo entry
repo_entry = {"name": repo_name, "url": repo_url}
# Check if repo already exists
existing = [r for r in config[category] if r.get("name") == repo_name]
if existing:
click.echo(f"โ Repository '{repo_name}' already exists in category '{category}'")
return
# Add the new repository
config[category].append(repo_entry)
# Write back to file
try:
with open(config_file, "w") as f:
yaml.dump(config, f, default_flow_style=False, sort_keys=False)
click.echo(f"โ
Added {repo_name} to {category} category")
click.echo("๐ Run 'nancy-brain build --force-update' to fetch the new repository")
except Exception as e:
click.echo(f"โ Error writing to {config_file}: {e}")
@cli.command()
@click.argument("article_url")
@click.argument("article_name")
@click.option("--category", default="articles", help="Category to add article to")
@click.option("--description", help="Description of the article")
def add_article(article_url, article_name, category, description):
"""Add a PDF article to the configuration."""
config_file = Path("config/articles.yml")
# Create articles config if it doesn't exist
if not config_file.exists():
config_file.parent.mkdir(parents=True, exist_ok=True)
articles_config = {}
else:
try:
with open(config_file, "r") as f:
articles_config = yaml.safe_load(f) or {}
except Exception as e:
click.echo(f"โ Error reading {config_file}: {e}")
return
# Add category if it doesn't exist
if category not in articles_config:
articles_config[category] = []
# Create article entry
article_entry = {"name": article_name, "url": article_url}
if description:
article_entry["description"] = description
# Check if article already exists
existing = [a for a in articles_config[category] if a.get("name") == article_name]
if existing:
click.echo(f"โ Article '{article_name}' already exists in category '{category}'")
return
# Add the new article
articles_config[category].append(article_entry)
# Write back to file
try:
with open(config_file, "w") as f:
yaml.dump(articles_config, f, default_flow_style=False, sort_keys=False)
click.echo(f"โ
Added article '{article_name}' to category '{category}'")
click.echo(f"๐ Run 'nancy-brain build --articles-config {config_file}' to index the new article")
except Exception as e:
click.echo(f"โ Error writing to {config_file}: {e}")
def _print_import_summary(source: str, summary: dict):
"""Print a consistent summary for import commands."""
click.echo(f"โ
{source} import complete")
click.echo(f" Added: {summary.get('added', 0)}")
click.echo(f" Skipped duplicates: {summary.get('skipped_duplicate', 0)}")
click.echo(f" Skipped (no URL): {summary.get('skipped_no_url', 0)}")
errors = summary.get("errors") or []
if errors:
click.echo(f" Errors: {len(errors)}")
for err in errors:
click.echo(f" - {err}")
@cli.command("import-bibtex")
@click.option(
"-f",
"--file",
"bib_file",
required=True,
type=click.Path(exists=True),
help="Path to the .bib file to import",
)
@click.option(
"--category",
default="journal_articles",
help="Category key in articles.yml (default: journal_articles)",
)
@click.option(
"--output",
default="config/articles.yml",
help="Path to articles.yml to update (default: config/articles.yml)",
)
@click.option("--dry-run", is_flag=True, help="Show what would be added without writing")
def import_bibtex(bib_file, category, output, dry_run):
"""Import articles from a BibTeX file into articles.yml."""
from nancy_brain.article_import import import_from_bibtex
try:
summary = import_from_bibtex(
bib_path=bib_file,
category=category,
output_path=output,
dry_run=dry_run,
)
_print_import_summary("BibTeX", summary)
except click.ClickException:
raise
except Exception as exc:
raise click.ClickException(f"BibTeX import failed: {exc}") from exc
@cli.command("import-ads")
@click.option("--library", required=True, help="Name of the ADS private library to import")
@click.option(
"--category",
default="journal_articles",
help="Category key in articles.yml (default: journal_articles)",
)
@click.option(
"--output",
default="config/articles.yml",
help="Path to articles.yml to update (default: config/articles.yml)",
)
@click.option("--dry-run", is_flag=True, help="Show what would be added without writing")
def import_ads(library, category, output, dry_run):
"""Import articles from an ADS private library into articles.yml."""
from nancy_brain.article_import import import_from_ads
try:
summary = import_from_ads(
library_name=library,
category=category,
output_path=output,
dry_run=dry_run,
)
_print_import_summary("ADS", summary)
except click.ClickException:
raise
except Exception as exc:
raise click.ClickException(f"ADS import failed: {exc}") from exc
@cli.command("add-new-user")
@click.argument("username")
@click.argument("password")
def add_new_user(username: str, password: str):
"""Create a new Nancy Brain login user (uses NB_USERS_DB)."""
try:
from connectors.http_api import auth
except ImportError as exc:
click.echo(f"โ Failed to import auth module: {exc}")
sys.exit(1)
try:
auth.create_user_table()
auth.add_user(username, password)
except Exception as exc:
click.echo(f"โ Failed to add user '{username}': {exc}")
sys.exit(1)
click.echo(f"โ
Added user '{username}'")
if __name__ == "__main__":
cli()
#!/usr/bin/env python3
"""Nancy Brain CLI interface."""
import click
import json
import logging
import os
import sys
import subprocess
import yaml
from pathlib import Path
# Avoid OpenMP duplicate runtime crashes by allowing duplicate lib loading when not set externally.
# Respect an existing environment setting (e.g., in a .env) but default to TRUE for CLI runs.
os.environ.setdefault("KMP_DUPLICATE_LIB_OK", "TRUE")
# Optional rich integration for colored output and spinners
try:
from rich.console import Console
from rich.spinner import Spinner
from rich.tree import Tree
from rich.panel import Panel
from rich.table import Table
from rich.text import Text
RICH_AVAILABLE = True
_console = Console()
except Exception:
RICH_AVAILABLE = False
_console = None
# Add the package root to sys.path to handle relative imports
package_root = Path(__file__).parent.parent
sys.path.insert(0, str(package_root))
from nancy_brain.config_validation import (
validate_repositories_config,
validate_articles_config,
)
# Get version from package
try:
from nancy_brain import __version__
except ImportError:
__version__ = "unknown"
def cli():
"""Nancy Brain - Turn GitHub repos into AI-searchable knowledge bases."""
pass
def _configure_cli_logging() -> None:
"""Ensure OCR commands emit INFO logs to stderr for batch-job observability."""
root_logger = logging.getLogger()
if not root_logger.handlers:
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
else:
root_logger.setLevel(logging.INFO)
def init(project_name):
"""Initialize a new Nancy Brain project.
This command creates a minimal `config/` directory with a `repositories.yml`
file to get you started. Edit the file and then run `nancy-brain build`.
"""
project_path = Path(project_name)
project_path.mkdir(exist_ok=True)
# Create basic config structure
config_dir = project_path / "config"
config_dir.mkdir(exist_ok=True)
# Basic repositories.yml
repos_config = config_dir / "repositories.yml"
repos_config.write_text(
"""# Add your repositories here
# example_tools:
# - name: example-repo
# url: https://github.com/org/example-repo.git
"""
)
click.echo(f"โ
Initialized Nancy Brain project in {project_name}/")
click.echo(f"๐ Edit {repos_config} to add repositories")
click.echo("๐๏ธ Run 'nancy-brain build' to create the knowledge base")
Xet Storage Details
- Size:
- 38.3 kB
- Xet hash:
- a66f4d50222618298456ad1d64c0d87bb1391d7372029c7078c2a93e94bd8e80
ยท
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.