Download backend/app/qualifier_agent.py from Seth0330/qualifier: direct link, hf CLI and curl.
- Browser
- Download file 10.8 kB
-
https://huggingface.co/spaces/Seth0330/qualifier/resolve/main/backend/app/qualifier_agent.py
- Command line
-
hf download hf://spaces/Seth0330/qualifier/backend/app/qualifier_agent.py
-
curl -L -o qualifier_agent.py https://huggingface.co/spaces/Seth0330/qualifier/resolve/main/backend/app/qualifier_agent.py
10.8 kB
| """ | |
| The Qualifier's agentic decision loop: skill instructions + candidate text (from | |
| qualifier_extract.py) -> bounded tool-calling rounds against search_pricelist -> a forced final | |
| tool call that produces the schema-validated QualifierDecision. | |
| Same agentic tool-calling shape as the chat agent's stream_agentic_reply_openai (bounded rounds, | |
| tool results fed back as messages), but non-streaming (this is a background run, not a live reply) | |
| and terminating in a structured object instead of prose. | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import os | |
| from typing import Any, Dict, List, Optional, Tuple | |
| from sqlalchemy.orm import Session | |
| from .database import QualifierSkill | |
| from .openai_client_factory import get_chat_client | |
| from .qualifier_retrieval import search_pricelist | |
| MAX_LOOKUP_ROUNDS = 4 | |
| QUALIFIER_CHAT_MODEL = os.getenv("QUALIFIER_CHAT_MODEL", "gpt-5.4-nano").strip() or "gpt-5.4-nano" | |
| _SEARCH_PRICELIST_TOOL = { | |
| "type": "function", | |
| "function": { | |
| "name": "search_pricelist", | |
| "description": ( | |
| "Search FTL's Wittur pricelist (the agent's only knowledge base) for a catalog match " | |
| "to an RFQ line item. Also searches any indexed skill reference material. Call this " | |
| "once per distinct item you need to classify — don't guess at a match without checking." | |
| ), | |
| "parameters": { | |
| "type": "object", | |
| "properties": { | |
| "query": { | |
| "type": "string", | |
| "description": "A short description of the item to look up, e.g. 'center opening door operator 42 inch GAL' or 'counterweight roller guide 3 inch rail'.", | |
| } | |
| }, | |
| "required": ["query"], | |
| }, | |
| }, | |
| } | |
| _SUBMIT_DECISION_TOOL = { | |
| "type": "function", | |
| "function": { | |
| "name": "submit_qualification_decision", | |
| "description": "Submit your final qualification decision for this RFQ. Always call this exactly once, as your last action.", | |
| "parameters": { | |
| "type": "object", | |
| "properties": { | |
| "qualify": {"type": "string", "enum": ["qualify", "disqualify", "needs_review"]}, | |
| "project_type": {"type": "string", "enum": ["modernization", "new_construction", "unknown"]}, | |
| "matched_items": { | |
| "type": "array", | |
| "items": { | |
| "type": "object", | |
| "properties": { | |
| "item": {"type": "string"}, | |
| "category": {"type": "string"}, | |
| "match": {"type": "string", "enum": ["exact", "ambiguous"]}, | |
| "catalog_ref": {"type": ["string", "null"]}, | |
| "note": {"type": "string"}, | |
| }, | |
| "required": ["item", "category", "match"], | |
| }, | |
| }, | |
| "excluded_items": { | |
| "type": "array", | |
| "items": { | |
| "type": "object", | |
| "properties": {"item": {"type": "string"}, "reason": {"type": "string"}}, | |
| "required": ["item", "reason"], | |
| }, | |
| }, | |
| "flags": {"type": "array", "items": {"type": "string"}}, | |
| "deadline": {"type": ["string", "null"]}, | |
| "project_name": {"type": "string"}, | |
| "reasoning": {"type": "string"}, | |
| "confidence": {"type": "number", "minimum": 0, "maximum": 1}, | |
| }, | |
| "required": ["qualify", "project_type", "matched_items", "excluded_items", "flags", "reasoning", "confidence"], | |
| }, | |
| }, | |
| } | |
| def build_system_prompt(skill: QualifierSkill) -> str: | |
| instructions = (skill.instructions or "").strip() | |
| return ( | |
| instructions | |
| + "\n\n---\n\n" | |
| + "You will be given the candidate text extracted from one RFQ (email + targeted spec " | |
| "excerpts). Use the search_pricelist tool to check any item you're unsure matches FTL's " | |
| "catalog before deciding. When you're done, call submit_qualification_decision exactly " | |
| "once with your full structured decision — do not respond with plain text." | |
| ) | |
| def _run_tool_call(db: Session, tenant_id: int, name: str, arguments: Dict[str, Any]) -> str: | |
| if name == "search_pricelist": | |
| query = arguments.get("query", "") | |
| results = search_pricelist(db, tenant_id, query, top_k=5) | |
| if not results: | |
| return json.dumps({"results": [], "note": "No pricelist matches found for this query."}) | |
| return json.dumps({"results": results}) | |
| return json.dumps({"error": f"Unknown tool: {name}"}) | |
| def run_qualification( | |
| db: Session, | |
| tenant_id: int, | |
| skill: QualifierSkill, | |
| candidate_text: str, | |
| ) -> Tuple[Dict[str, Any], int]: | |
| """Runs the bounded agentic loop to completion and returns (decision_dict, total_tokens). | |
| Raises RuntimeError if the model never calls submit_qualification_decision within the round | |
| cap — callers should catch this and record the run as an error, not silently return nothing.""" | |
| client = get_chat_client() | |
| system_prompt = build_system_prompt(skill) | |
| messages: List[Dict[str, Any]] = [ | |
| {"role": "system", "content": system_prompt}, | |
| {"role": "user", "content": candidate_text}, | |
| ] | |
| tools = [_SEARCH_PRICELIST_TOOL, _SUBMIT_DECISION_TOOL] | |
| total_tokens = 0 | |
| for round_num in range(MAX_LOOKUP_ROUNDS + 1): | |
| force_final = round_num == MAX_LOOKUP_ROUNDS | |
| resp = client.chat.completions.create( | |
| model=QUALIFIER_CHAT_MODEL, | |
| messages=messages, | |
| tools=tools, | |
| tool_choice=( | |
| {"type": "function", "function": {"name": "submit_qualification_decision"}} | |
| if force_final | |
| else "auto" | |
| ), | |
| ) | |
| usage = getattr(resp, "usage", None) | |
| if usage is not None: | |
| total_tokens += int(getattr(usage, "total_tokens", 0) or 0) | |
| choice = resp.choices[0] | |
| msg = choice.message | |
| tool_calls = getattr(msg, "tool_calls", None) or [] | |
| if not tool_calls: | |
| # Model replied with plain text instead of calling a tool — nudge it back on track | |
| # rather than failing the run outright, since a nano-tier model is more likely to drift. | |
| messages.append({"role": "assistant", "content": msg.content or ""}) | |
| messages.append( | |
| { | |
| "role": "user", | |
| "content": "Please respond only via a tool call — either search_pricelist or, if you're ready, submit_qualification_decision.", | |
| } | |
| ) | |
| continue | |
| messages.append( | |
| { | |
| "role": "assistant", | |
| "content": msg.content, | |
| "tool_calls": [ | |
| { | |
| "id": tc.id, | |
| "type": "function", | |
| "function": {"name": tc.function.name, "arguments": tc.function.arguments}, | |
| } | |
| for tc in tool_calls | |
| ], | |
| } | |
| ) | |
| decision_call = next((tc for tc in tool_calls if tc.function.name == "submit_qualification_decision"), None) | |
| if decision_call is not None: | |
| try: | |
| decision = json.loads(decision_call.function.arguments) | |
| except json.JSONDecodeError as e: | |
| raise RuntimeError(f"Model returned invalid JSON for its decision: {e}") from e | |
| return decision, total_tokens | |
| for tc in tool_calls: | |
| try: | |
| args = json.loads(tc.function.arguments or "{}") | |
| except json.JSONDecodeError: | |
| args = {} | |
| result = _run_tool_call(db, tenant_id, tc.function.name, args) | |
| messages.append({"role": "tool", "tool_call_id": tc.id, "content": result}) | |
| raise RuntimeError( | |
| f"Model did not submit a decision within {MAX_LOOKUP_ROUNDS} tool-call rounds." | |
| ) | |
| # --------------------------------------------------------------------------- | |
| # Evals — a qualifier eval re-runs the FULL pipeline against a previously-uploaded run's stored | |
| # input (see qualifier_routes.py's evals/run handler) and checks the fresh decision, rather than | |
| # probing a single prompt/response the way the chat agent's evals do. | |
| # --------------------------------------------------------------------------- | |
| def check_eval_assertion(eval_item: Dict[str, Any], decision: Dict[str, Any]) -> Tuple[bool, str]: | |
| assertion_type = eval_item.get("assertion_type", "decision_equals") | |
| expected_decision = eval_item.get("expected_decision") | |
| expected_project_type = eval_item.get("expected_project_type") | |
| if assertion_type == "decision_equals": | |
| ok = True | |
| details = [] | |
| if expected_decision and decision.get("qualify") != expected_decision: | |
| ok = False | |
| details.append(f"expected qualify={expected_decision}, got {decision.get('qualify')}") | |
| if expected_project_type and decision.get("project_type") != expected_project_type: | |
| ok = False | |
| details.append(f"expected project_type={expected_project_type}, got {decision.get('project_type')}") | |
| return ok, "; ".join(details) if details else "matched expected decision/project_type" | |
| reasoning = (decision.get("reasoning") or "") | |
| haystack = json.dumps(decision).lower() | |
| needle = (eval_item.get("assertion_value") or "").lower() | |
| if assertion_type == "must_contain": | |
| ok = needle in haystack | |
| return ok, "found" if ok else f"'{needle}' not found in decision output" | |
| if assertion_type == "must_not_contain": | |
| ok = needle not in haystack | |
| return ok, "absent as expected" if ok else f"'{needle}' unexpectedly found in decision output" | |
| if assertion_type == "llm_judge": | |
| client = get_chat_client() | |
| prompt = ( | |
| f"Decision output:\n{json.dumps(decision, indent=2)}\n\n" | |
| f"Rubric: {eval_item.get('assertion_value')}\n\n" | |
| "Does the decision output satisfy the rubric? Answer with exactly PASS or FAIL, then a one-sentence reason." | |
| ) | |
| resp = client.chat.completions.create( | |
| model=QUALIFIER_CHAT_MODEL, | |
| messages=[{"role": "user", "content": prompt}], | |
| temperature=0, | |
| ) | |
| text = (resp.choices[0].message.content or "").strip() | |
| ok = text.upper().startswith("PASS") | |
| return ok, text | |
| return False, f"Unknown assertion_type: {assertion_type}" | |