File size: 3,401 Bytes
f8b48da | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 | """Live smoke test for the agentic tool-use path (needs infra + OpenRouter key).
Runs :func:`run_agent` against the real stack — Postgres, Qdrant, and
nex-n2.5-mini via OpenRouter — and asserts the model actually performed
function calls (not a plain completion) and returned cited evidence.
Usage:
python scripts/agent_smoke.py [--title-substr PROPOSAL] [--question "..."]
"""
from __future__ import annotations
import argparse
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from app.config import get_settings # noqa: E402
from app.db.models import Document # noqa: E402
from app.db.session import SessionLocal # noqa: E402
from app.generation.agent import run_agent # noqa: E402
from sqlalchemy import select # noqa: E402
def find_document_id(title_substr: str) -> tuple[str, str]:
"""Return (document_id, title) of the newest doc whose title matches."""
with SessionLocal() as session:
rows = session.execute(select(Document).order_by(Document.created_at.desc())).scalars().all()
for doc in rows:
if title_substr.lower() in (doc.title or "").lower():
return doc.document_id, doc.title
raise SystemExit(f"No document with {title_substr!r} in its title.")
def main() -> int:
parser = argparse.ArgumentParser(description="Live agent tool-use smoke test")
parser.add_argument("--title-substr", default="Proposal")
parser.add_argument(
"--question",
default="What is this document about? Summarize its main topic.",
)
args = parser.parse_args()
settings = get_settings()
if not settings.openrouter_api_key:
print("FAIL: OPENROUTER_API_KEY is not set.")
return 2
print(f"Model: {settings.openrouter_model}")
document_id, title = find_document_id(args.title_substr)
print(f"Document: {title} ({document_id})")
print(f"Question: {args.question}")
print("-" * 70)
result = run_agent(args.question, document_ids=[document_id], top_k=5, max_steps=6)
for i, step in enumerate(result.steps, 1):
status = "ok" if step.ok else "ERROR"
print(f"[step {i}] {step.tool_name} {status} ({step.latency_ms}ms)")
print(f" args: {step.arguments}")
print(f" output: {step.output_preview[:200]}")
print("-" * 70)
print(f"finish_reason: {result.finish_reason}")
print(f"evidence chunks: {len(result.evidence)}")
print(f"answer: {result.answer[:800]}")
print("-" * 70)
failures = []
if not result.steps:
failures.append("model made zero tool calls (plain completion, not tool use)")
if not any(s.tool_name == "search_documents" and s.ok for s in result.steps):
failures.append("no successful search_documents call in the trace")
if not result.evidence:
failures.append("no evidence chunks retrieved")
if not result.answer.strip():
failures.append("empty final answer")
if failures:
print("SMOKE RESULT: FAIL")
for failure in failures:
print(f" - {failure}")
return 1
print(
f"SMOKE RESULT: PASS "
f"({len(result.steps)} tool calls, "
f"{len(result.evidence)} evidence chunks, "
f"{result.latency_ms / 1000:.1f}s)"
)
return 0
if __name__ == "__main__":
sys.exit(main())
|