""" The Qualifier's agentic decision loop: skill instructions + candidate text (from qualifier_extract.py) -> bounded tool-calling rounds against search_pricelist -> a forced final tool call that produces the schema-validated QualifierDecision. Same agentic tool-calling shape as the chat agent's stream_agentic_reply_openai (bounded rounds, tool results fed back as messages), but non-streaming (this is a background run, not a live reply) and terminating in a structured object instead of prose. """ from __future__ import annotations import json import os from typing import Any, Dict, List, Optional, Tuple from sqlalchemy.orm import Session from .database import QualifierSkill from .openai_client_factory import get_chat_client from .qualifier_retrieval import search_pricelist MAX_LOOKUP_ROUNDS = 4 QUALIFIER_CHAT_MODEL = os.getenv("QUALIFIER_CHAT_MODEL", "gpt-5.4-nano").strip() or "gpt-5.4-nano" _SEARCH_PRICELIST_TOOL = { "type": "function", "function": { "name": "search_pricelist", "description": ( "Search FTL's Wittur pricelist (the agent's only knowledge base) for a catalog match " "to an RFQ line item. Also searches any indexed skill reference material. Call this " "once per distinct item you need to classify — don't guess at a match without checking." ), "parameters": { "type": "object", "properties": { "query": { "type": "string", "description": "A short description of the item to look up, e.g. 'center opening door operator 42 inch GAL' or 'counterweight roller guide 3 inch rail'.", } }, "required": ["query"], }, }, } _SUBMIT_DECISION_TOOL = { "type": "function", "function": { "name": "submit_qualification_decision", "description": "Submit your final qualification decision for this RFQ. Always call this exactly once, as your last action.", "parameters": { "type": "object", "properties": { "qualify": {"type": "string", "enum": ["qualify", "disqualify", "needs_review"]}, "project_type": {"type": "string", "enum": ["modernization", "new_construction", "unknown"]}, "matched_items": { "type": "array", "items": { "type": "object", "properties": { "item": {"type": "string"}, "category": {"type": "string"}, "match": {"type": "string", "enum": ["exact", "ambiguous"]}, "catalog_ref": {"type": ["string", "null"]}, "note": {"type": "string"}, }, "required": ["item", "category", "match"], }, }, "excluded_items": { "type": "array", "items": { "type": "object", "properties": {"item": {"type": "string"}, "reason": {"type": "string"}}, "required": ["item", "reason"], }, }, "flags": {"type": "array", "items": {"type": "string"}}, "deadline": {"type": ["string", "null"]}, "project_name": {"type": "string"}, "reasoning": {"type": "string"}, "confidence": {"type": "number", "minimum": 0, "maximum": 1}, }, "required": ["qualify", "project_type", "matched_items", "excluded_items", "flags", "reasoning", "confidence"], }, }, } def build_system_prompt(skill: QualifierSkill) -> str: instructions = (skill.instructions or "").strip() return ( instructions + "\n\n---\n\n" + "You will be given the candidate text extracted from one RFQ (email + targeted spec " "excerpts). Use the search_pricelist tool to check any item you're unsure matches FTL's " "catalog before deciding. When you're done, call submit_qualification_decision exactly " "once with your full structured decision — do not respond with plain text." ) def _run_tool_call(db: Session, tenant_id: int, name: str, arguments: Dict[str, Any]) -> str: if name == "search_pricelist": query = arguments.get("query", "") results = search_pricelist(db, tenant_id, query, top_k=5) if not results: return json.dumps({"results": [], "note": "No pricelist matches found for this query."}) return json.dumps({"results": results}) return json.dumps({"error": f"Unknown tool: {name}"}) def run_qualification( db: Session, tenant_id: int, skill: QualifierSkill, candidate_text: str, ) -> Tuple[Dict[str, Any], int]: """Runs the bounded agentic loop to completion and returns (decision_dict, total_tokens). Raises RuntimeError if the model never calls submit_qualification_decision within the round cap — callers should catch this and record the run as an error, not silently return nothing.""" client = get_chat_client() system_prompt = build_system_prompt(skill) messages: List[Dict[str, Any]] = [ {"role": "system", "content": system_prompt}, {"role": "user", "content": candidate_text}, ] tools = [_SEARCH_PRICELIST_TOOL, _SUBMIT_DECISION_TOOL] total_tokens = 0 for round_num in range(MAX_LOOKUP_ROUNDS + 1): force_final = round_num == MAX_LOOKUP_ROUNDS resp = client.chat.completions.create( model=QUALIFIER_CHAT_MODEL, messages=messages, tools=tools, tool_choice=( {"type": "function", "function": {"name": "submit_qualification_decision"}} if force_final else "auto" ), ) usage = getattr(resp, "usage", None) if usage is not None: total_tokens += int(getattr(usage, "total_tokens", 0) or 0) choice = resp.choices[0] msg = choice.message tool_calls = getattr(msg, "tool_calls", None) or [] if not tool_calls: # Model replied with plain text instead of calling a tool — nudge it back on track # rather than failing the run outright, since a nano-tier model is more likely to drift. messages.append({"role": "assistant", "content": msg.content or ""}) messages.append( { "role": "user", "content": "Please respond only via a tool call — either search_pricelist or, if you're ready, submit_qualification_decision.", } ) continue messages.append( { "role": "assistant", "content": msg.content, "tool_calls": [ { "id": tc.id, "type": "function", "function": {"name": tc.function.name, "arguments": tc.function.arguments}, } for tc in tool_calls ], } ) decision_call = next((tc for tc in tool_calls if tc.function.name == "submit_qualification_decision"), None) if decision_call is not None: try: decision = json.loads(decision_call.function.arguments) except json.JSONDecodeError as e: raise RuntimeError(f"Model returned invalid JSON for its decision: {e}") from e return decision, total_tokens for tc in tool_calls: try: args = json.loads(tc.function.arguments or "{}") except json.JSONDecodeError: args = {} result = _run_tool_call(db, tenant_id, tc.function.name, args) messages.append({"role": "tool", "tool_call_id": tc.id, "content": result}) raise RuntimeError( f"Model did not submit a decision within {MAX_LOOKUP_ROUNDS} tool-call rounds." ) # --------------------------------------------------------------------------- # Evals — a qualifier eval re-runs the FULL pipeline against a previously-uploaded run's stored # input (see qualifier_routes.py's evals/run handler) and checks the fresh decision, rather than # probing a single prompt/response the way the chat agent's evals do. # --------------------------------------------------------------------------- def check_eval_assertion(eval_item: Dict[str, Any], decision: Dict[str, Any]) -> Tuple[bool, str]: assertion_type = eval_item.get("assertion_type", "decision_equals") expected_decision = eval_item.get("expected_decision") expected_project_type = eval_item.get("expected_project_type") if assertion_type == "decision_equals": ok = True details = [] if expected_decision and decision.get("qualify") != expected_decision: ok = False details.append(f"expected qualify={expected_decision}, got {decision.get('qualify')}") if expected_project_type and decision.get("project_type") != expected_project_type: ok = False details.append(f"expected project_type={expected_project_type}, got {decision.get('project_type')}") return ok, "; ".join(details) if details else "matched expected decision/project_type" reasoning = (decision.get("reasoning") or "") haystack = json.dumps(decision).lower() needle = (eval_item.get("assertion_value") or "").lower() if assertion_type == "must_contain": ok = needle in haystack return ok, "found" if ok else f"'{needle}' not found in decision output" if assertion_type == "must_not_contain": ok = needle not in haystack return ok, "absent as expected" if ok else f"'{needle}' unexpectedly found in decision output" if assertion_type == "llm_judge": client = get_chat_client() prompt = ( f"Decision output:\n{json.dumps(decision, indent=2)}\n\n" f"Rubric: {eval_item.get('assertion_value')}\n\n" "Does the decision output satisfy the rubric? Answer with exactly PASS or FAIL, then a one-sentence reason." ) resp = client.chat.completions.create( model=QUALIFIER_CHAT_MODEL, messages=[{"role": "user", "content": prompt}], temperature=0, ) text = (resp.choices[0].message.content or "").strip() ok = text.upper().startswith("PASS") return ok, text return False, f"Unknown assertion_type: {assertion_type}"