"""Command-line interface: ingest, ask, eval, serve.""" from __future__ import annotations import json import sys from pathlib import Path import typer from .config import settings from .engine import RagEngine from .eval import evaluate from .eval.harness import load_cases from .ingestion.loaders import _LOADERS, load_file from .retrieval import Retriever # Ensure Unicode output (arrows, box-drawing) doesn't crash on Windows' cp1252 console. for _stream in (sys.stdout, sys.stderr): try: _stream.reconfigure(encoding="utf-8") # type: ignore[union-attr] except (AttributeError, ValueError): pass app = typer.Typer(help="Lumen RAG — ingest, ask, and evaluate a RAG pipeline.") @app.command() def ingest( paths: list[str] = typer.Argument(..., help="Files or directories (.txt/.md/.html/.pdf/.docx/.pptx)."), chunk_size: int = 120, overlap: int = 20, ) -> None: """Index documents into the persistent vector store.""" files: list[Path] = [] for p in paths: path = Path(p) if path.is_dir(): for ext in _LOADERS: files.extend(path.rglob(f"*{ext}")) else: files.append(path) engine = ( RagEngine.load() if Path(settings.index_dir, "chunks.json").exists() else RagEngine() ) # Go through the loaders so PDF/DOCX/HTML (and Docling, when enabled) are # parsed rather than read as raw bytes. docs = [load_file(f) for f in files] total = engine.add_documents(docs, chunk_size=chunk_size, overlap=overlap) engine.save() typer.echo(f"Indexed {len(files)} file(s) → {total} chunks in {settings.index_dir}/") @app.command() def ask( question: str, k: int = 5, mode: str = typer.Option("hybrid", help="Retrieval mode: vector | bm25 | hybrid"), ) -> None: """Query the index and print an answer with citations.""" engine = RagEngine.load() result = engine.query(question, k=k, mode=mode) # type: ignore[arg-type] typer.echo("\n" + result.text + "\n") if result.confidence is not None: typer.echo(f"Retrieval confidence: {result.confidence:.3f}" + (" (abstained)" if result.abstained else "")) typer.echo("Sources:") for c in result.citations: typer.echo(f" [{c['n']}] {c['doc_id']} (score={c['score']})") @app.command(name="eval") def run_eval(dataset: str, k: int = 5) -> None: """Score the retriever against a JSONL of labelled questions.""" engine = RagEngine.load() cases = load_cases(dataset) report = evaluate(Retriever(engine.store, engine.embedder), cases, k=k) typer.echo("\n" + report.pretty() + "\n") typer.echo(json.dumps(report.as_dict())) @app.command(name="eval-answers") def run_answer_eval( dataset: str, k: int = 5, mode: str = typer.Option("hybrid", help="Retrieval mode: vector | bm25 | hybrid"), judge: str = typer.Option("gpt-4o-mini", help="Judge model for DeepEval metrics."), threshold: float = typer.Option(0.5, help="Per-metric pass threshold (0-1)."), out: str = typer.Option("", help="Optional path to write per-case JSON results."), claims: bool = typer.Option( False, "--claims", help="Claim-level diagnostics instead of an LLM judge: split each answer into claims, " "align them with the retrieved chunks, and attribute failures to retrieval or generation. " "Works offline.", ), tau: float = typer.Option(0.6, help="With --claims: alignment score a chunk needs to ground a claim (0-1)."), abstain_threshold: float | None = typer.Option( None, help="With --claims: refuse when retrieval confidence is below this " "(overrides LUMEN_ABSTAIN_THRESHOLD; LUMEN_READER also applies).", ), ) -> None: """Judge generated answers (DeepEval), or with --claims attribute failures to retrieval vs generation.""" engine = RagEngine.load() cases = load_cases(dataset) if claims: from .eval.claims import diagnose_claims if abstain_threshold is not None: settings.abstain_threshold = abstain_threshold # The engine's own path, so both abstention gates and reader answers are measured. claim_report = diagnose_claims( engine.retriever, cases, k=k, mode=mode, # type: ignore[arg-type] tau=tau, pipeline=lambda q: engine.answer_with_context(q, k=k, mode=mode), # type: ignore[arg-type] ) typer.echo("\n" + claim_report.pretty() + "\n") typer.echo(json.dumps(claim_report.as_dict())) if out: Path(out).write_text(json.dumps(claim_report.per_case, indent=2), encoding="utf-8") typer.echo(f"Per-case results → {out}") return from .eval.answer_quality import evaluate_answers try: report = evaluate_answers( engine, cases, k=k, mode=mode, threshold=threshold, judge_model=judge # type: ignore[arg-type] ) except (ImportError, RuntimeError) as e: typer.echo(f"error: {e}", err=True) raise typer.Exit(code=1) from e typer.echo("\n" + report.pretty() + "\n") typer.echo(json.dumps(report.as_dict())) if out: Path(out).write_text(json.dumps(report.per_case, indent=2), encoding="utf-8") typer.echo(f"Per-case results → {out}") @app.command() def serve(host: str = "0.0.0.0", port: int = 8000) -> None: """Run the FastAPI server.""" import uvicorn uvicorn.run("lumen_rag.api.app:app", host=host, port=port, reload=False) if __name__ == "__main__": app()