qualifier / backend /app /qualifier_agent.py
Sethu0330's picture
Add FTL RFQ Qualifier agent + full SALES CRM app for HF Spaces testing
005df22 unverified
Raw History Blame Contribute Delete
10.8 kB
"""
The Qualifier's agentic decision loop: skill instructions + candidate text (from
qualifier_extract.py) -> bounded tool-calling rounds against search_pricelist -> a forced final
tool call that produces the schema-validated QualifierDecision.
Same agentic tool-calling shape as the chat agent's stream_agentic_reply_openai (bounded rounds,
tool results fed back as messages), but non-streaming (this is a background run, not a live reply)
and terminating in a structured object instead of prose.
"""
from __future__ import annotations
import json
import os
from typing import Any, Dict, List, Optional, Tuple
from sqlalchemy.orm import Session
from .database import QualifierSkill
from .openai_client_factory import get_chat_client
from .qualifier_retrieval import search_pricelist
MAX_LOOKUP_ROUNDS = 4
QUALIFIER_CHAT_MODEL = os.getenv("QUALIFIER_CHAT_MODEL", "gpt-5.4-nano").strip() or "gpt-5.4-nano"
_SEARCH_PRICELIST_TOOL = {
"type": "function",
"function": {
"name": "search_pricelist",
"description": (
"Search FTL's Wittur pricelist (the agent's only knowledge base) for a catalog match "
"to an RFQ line item. Also searches any indexed skill reference material. Call this "
"once per distinct item you need to classify — don't guess at a match without checking."
),
"parameters": {
"type": "object",
"properties": {
"query": {
"type": "string",
"description": "A short description of the item to look up, e.g. 'center opening door operator 42 inch GAL' or 'counterweight roller guide 3 inch rail'.",
}
},
"required": ["query"],
},
},
}
_SUBMIT_DECISION_TOOL = {
"type": "function",
"function": {
"name": "submit_qualification_decision",
"description": "Submit your final qualification decision for this RFQ. Always call this exactly once, as your last action.",
"parameters": {
"type": "object",
"properties": {
"qualify": {"type": "string", "enum": ["qualify", "disqualify", "needs_review"]},
"project_type": {"type": "string", "enum": ["modernization", "new_construction", "unknown"]},
"matched_items": {
"type": "array",
"items": {
"type": "object",
"properties": {
"item": {"type": "string"},
"category": {"type": "string"},
"match": {"type": "string", "enum": ["exact", "ambiguous"]},
"catalog_ref": {"type": ["string", "null"]},
"note": {"type": "string"},
},
"required": ["item", "category", "match"],
},
},
"excluded_items": {
"type": "array",
"items": {
"type": "object",
"properties": {"item": {"type": "string"}, "reason": {"type": "string"}},
"required": ["item", "reason"],
},
},
"flags": {"type": "array", "items": {"type": "string"}},
"deadline": {"type": ["string", "null"]},
"project_name": {"type": "string"},
"reasoning": {"type": "string"},
"confidence": {"type": "number", "minimum": 0, "maximum": 1},
},
"required": ["qualify", "project_type", "matched_items", "excluded_items", "flags", "reasoning", "confidence"],
},
},
}
def build_system_prompt(skill: QualifierSkill) -> str:
instructions = (skill.instructions or "").strip()
return (
instructions
+ "\n\n---\n\n"
+ "You will be given the candidate text extracted from one RFQ (email + targeted spec "
"excerpts). Use the search_pricelist tool to check any item you're unsure matches FTL's "
"catalog before deciding. When you're done, call submit_qualification_decision exactly "
"once with your full structured decision — do not respond with plain text."
)
def _run_tool_call(db: Session, tenant_id: int, name: str, arguments: Dict[str, Any]) -> str:
if name == "search_pricelist":
query = arguments.get("query", "")
results = search_pricelist(db, tenant_id, query, top_k=5)
if not results:
return json.dumps({"results": [], "note": "No pricelist matches found for this query."})
return json.dumps({"results": results})
return json.dumps({"error": f"Unknown tool: {name}"})
def run_qualification(
db: Session,
tenant_id: int,
skill: QualifierSkill,
candidate_text: str,
) -> Tuple[Dict[str, Any], int]:
"""Runs the bounded agentic loop to completion and returns (decision_dict, total_tokens).
Raises RuntimeError if the model never calls submit_qualification_decision within the round
cap — callers should catch this and record the run as an error, not silently return nothing."""
client = get_chat_client()
system_prompt = build_system_prompt(skill)
messages: List[Dict[str, Any]] = [
{"role": "system", "content": system_prompt},
{"role": "user", "content": candidate_text},
]
tools = [_SEARCH_PRICELIST_TOOL, _SUBMIT_DECISION_TOOL]
total_tokens = 0
for round_num in range(MAX_LOOKUP_ROUNDS + 1):
force_final = round_num == MAX_LOOKUP_ROUNDS
resp = client.chat.completions.create(
model=QUALIFIER_CHAT_MODEL,
messages=messages,
tools=tools,
tool_choice=(
{"type": "function", "function": {"name": "submit_qualification_decision"}}
if force_final
else "auto"
),
)
usage = getattr(resp, "usage", None)
if usage is not None:
total_tokens += int(getattr(usage, "total_tokens", 0) or 0)
choice = resp.choices[0]
msg = choice.message
tool_calls = getattr(msg, "tool_calls", None) or []
if not tool_calls:
# Model replied with plain text instead of calling a tool — nudge it back on track
# rather than failing the run outright, since a nano-tier model is more likely to drift.
messages.append({"role": "assistant", "content": msg.content or ""})
messages.append(
{
"role": "user",
"content": "Please respond only via a tool call — either search_pricelist or, if you're ready, submit_qualification_decision.",
}
)
continue
messages.append(
{
"role": "assistant",
"content": msg.content,
"tool_calls": [
{
"id": tc.id,
"type": "function",
"function": {"name": tc.function.name, "arguments": tc.function.arguments},
}
for tc in tool_calls
],
}
)
decision_call = next((tc for tc in tool_calls if tc.function.name == "submit_qualification_decision"), None)
if decision_call is not None:
try:
decision = json.loads(decision_call.function.arguments)
except json.JSONDecodeError as e:
raise RuntimeError(f"Model returned invalid JSON for its decision: {e}") from e
return decision, total_tokens
for tc in tool_calls:
try:
args = json.loads(tc.function.arguments or "{}")
except json.JSONDecodeError:
args = {}
result = _run_tool_call(db, tenant_id, tc.function.name, args)
messages.append({"role": "tool", "tool_call_id": tc.id, "content": result})
raise RuntimeError(
f"Model did not submit a decision within {MAX_LOOKUP_ROUNDS} tool-call rounds."
)
# ---------------------------------------------------------------------------
# Evals — a qualifier eval re-runs the FULL pipeline against a previously-uploaded run's stored
# input (see qualifier_routes.py's evals/run handler) and checks the fresh decision, rather than
# probing a single prompt/response the way the chat agent's evals do.
# ---------------------------------------------------------------------------
def check_eval_assertion(eval_item: Dict[str, Any], decision: Dict[str, Any]) -> Tuple[bool, str]:
assertion_type = eval_item.get("assertion_type", "decision_equals")
expected_decision = eval_item.get("expected_decision")
expected_project_type = eval_item.get("expected_project_type")
if assertion_type == "decision_equals":
ok = True
details = []
if expected_decision and decision.get("qualify") != expected_decision:
ok = False
details.append(f"expected qualify={expected_decision}, got {decision.get('qualify')}")
if expected_project_type and decision.get("project_type") != expected_project_type:
ok = False
details.append(f"expected project_type={expected_project_type}, got {decision.get('project_type')}")
return ok, "; ".join(details) if details else "matched expected decision/project_type"
reasoning = (decision.get("reasoning") or "")
haystack = json.dumps(decision).lower()
needle = (eval_item.get("assertion_value") or "").lower()
if assertion_type == "must_contain":
ok = needle in haystack
return ok, "found" if ok else f"'{needle}' not found in decision output"
if assertion_type == "must_not_contain":
ok = needle not in haystack
return ok, "absent as expected" if ok else f"'{needle}' unexpectedly found in decision output"
if assertion_type == "llm_judge":
client = get_chat_client()
prompt = (
f"Decision output:\n{json.dumps(decision, indent=2)}\n\n"
f"Rubric: {eval_item.get('assertion_value')}\n\n"
"Does the decision output satisfy the rubric? Answer with exactly PASS or FAIL, then a one-sentence reason."
)
resp = client.chat.completions.create(
model=QUALIFIER_CHAT_MODEL,
messages=[{"role": "user", "content": prompt}],
temperature=0,
)
text = (resp.choices[0].message.content or "").strip()
ok = text.upper().startswith("PASS")
return ok, text
return False, f"Unknown assertion_type: {assertion_type}"