from __future__ import annotations import json import os from openai import OpenAI from dotenv import load_dotenv from support_triage_env.models import TriageAction from support_triage_env.server.environment import SupportTriageEnvironment from support_triage_env.tasks import task_ids load_dotenv() API_BASE_URL = os.getenv("API_BASE_URL", "https://api.openai.com/v1") MODEL_NAME = os.getenv("MODEL_NAME", "gpt-4.1-mini") HF_TOKEN = os.getenv("HF_TOKEN") REQUEST_TIMEOUT_S = float(os.getenv("REQUEST_TIMEOUT_S", "60")) TASK_IDS_OVERRIDE = os.getenv("TASK_IDS") if HF_TOKEN is None: raise ValueError("HF_TOKEN environment variable is required") if HF_TOKEN.startswith("hf_") and API_BASE_URL == "https://api.openai.com/v1": raise ValueError( "HF_TOKEN looks like a Hugging Face Hub token, but API_BASE_URL points to OpenAI. " "Use a provider API key compatible with API_BASE_URL, or change API_BASE_URL to the " "endpoint that expects this token." ) client = OpenAI(base_url=API_BASE_URL, api_key=HF_TOKEN, timeout=REQUEST_TIMEOUT_S) ACTION_TOOL = [ { "type": "function", "function": { "name": "choose_action", "description": "Choose the next environment action.", "parameters": { "type": "object", "properties": { "action_type": { "type": "string", "enum": [ "set_priority", "set_queue", "set_status", "add_tag", "remove_tag", "draft_reply", "submit", ], }, "value": { "type": ["string", "null"], "description": "Action value, or null for submit.", }, }, "required": ["action_type", "value"], "additionalProperties": False, }, }, } ] TASK_PLAYBOOKS = { "billing_duplicate_charge": { "priority": "high", "queue": "billing", "status": "pending_refund_review", "tags": ["billing", "refund"], "reply": ( "I am sorry about the duplicate charge. We will process a refund for the duplicate " "charge within 24 hours while keeping your subscription active." ), }, "shipping_damaged_replacement": { "priority": "medium", "queue": "logistics", "status": "awaiting_verification", "tags": ["damaged", "replacement"], "reply": ( "I am sorry the monitor stand arrived damaged. We can process a replacement as soon " "as you send a photo of the damage and confirm your shipping address." ), }, "account_takeover_fraud": { "priority": "urgent", "queue": "trust_and_safety", "status": "escalated_security", "tags": ["security", "fraud", "account_takeover"], "reply": ( "We have escalated the unauthorized order incident and are working to secure the " "account. We will help you reset the password as part of this security response." ), }, } def _one_line(value: str | None) -> str: if value is None: return "null" return " ".join(value.replace("\n", " ").split()) def _format_reward(value: float | None) -> str: return f"{(value or 0.0):.2f}" def _action_for_log(action: TriageAction) -> str: if action.action_type == "submit": return "submit()" return f"{action.action_type}({json.dumps(action.value)})" def _build_prompt(observation) -> str: return _build_prompt_with_history(observation, []) def _build_prompt_with_history(observation, recent_steps: list[dict[str, str]]) -> str: payload = { "instructions": observation.instructions, "ticket": observation.ticket.model_dump(), "policy_notes": observation.policy_notes, "reply_requirements": observation.reply_requirements, "workspace": observation.workspace.model_dump(), "allowed_values": observation.allowed_values, "available_actions": observation.available_actions, "completion_score": observation.completion_score, "missing_items": observation.missing_items, "last_action_error": observation.last_action_error, "steps_remaining": observation.steps_remaining, "recent_steps": recent_steps[-4:], } solved_items = [key for key, score in observation.score_breakdown.items() if score >= 0.999] avoid_pairs = [ step["action"] for step in recent_steps[-4:] if step.get("reward", "").startswith("-") ] return ( "You are operating a customer support triage environment.\n" "Choose the single best next action.\n" "Use the choose_action function.\n" "Set value to null only for submit.\n" f"Already solved areas: {solved_items}.\n" f"Do not repeat recently bad actions: {avoid_pairs}.\n" "Prefer editing an unsolved area instead of revisiting a solved one.\n" f"Observation:{json.dumps(payload, separators=(',', ':'))}" ) def _extract_json_object(text: str) -> str: stripped = text.strip() if stripped.startswith("```"): stripped = stripped.strip("`") if stripped.startswith("json"): stripped = stripped[4:].strip() start = stripped.find("{") end = stripped.rfind("}") if start != -1 and end != -1 and end >= start: return stripped[start : end + 1] return stripped def _normalize_choice(value: str | None) -> str: return (value or "").strip().lower() def _normalize_tag(value: str | None) -> str: normalized = _normalize_choice(value).replace("-", "_").replace(" ", "_") while "__" in normalized: normalized = normalized.replace("__", "_") return normalized def _reply_covers_keywords(value: str | None, keywords: list[str]) -> bool: normalized = _normalize_choice(value).replace("_", " ").replace("-", " ") return all(_normalize_choice(keyword).replace("_", " ").replace("-", " ") in normalized for keyword in keywords) def _parse_action(content: str) -> TriageAction: data = json.loads(_extract_json_object(content)) return TriageAction( action_type=data["action_type"], value=data.get("value"), ) def _parse_tool_action(response) -> TriageAction | None: message = response.choices[0].message tool_calls = getattr(message, "tool_calls", None) or [] if not tool_calls: return None first_call = tool_calls[0] function = getattr(first_call, "function", None) if function is None or function.name != "choose_action": return None arguments = function.arguments or "{}" data = json.loads(arguments) return TriageAction( action_type=data["action_type"], value=data.get("value"), ) def _request_action_with_tools(prompt: str) -> TriageAction | None: response = client.chat.completions.create( model=MODEL_NAME, messages=[{"role": "user", "content": prompt}], temperature=0, tools=ACTION_TOOL, tool_choice="required", ) action = _parse_tool_action(response) if action is not None: return action content = response.choices[0].message.content or "" return _parse_action(content) def _request_action_with_json(prompt: str) -> TriageAction: response = client.chat.completions.create( model=MODEL_NAME, messages=[ { "role": "user", "content": ( f"{prompt}\n" "Return only compact JSON with keys action_type and value." ), } ], temperature=0, ) content = response.choices[0].message.content or "" return _parse_action(content) def _planned_action(observation) -> TriageAction: playbook = TASK_PLAYBOOKS.get(observation.task_id) if playbook is None: if observation.workspace.priority is None: return TriageAction(action_type="set_priority", value="medium") if observation.workspace.queue is None: return TriageAction(action_type="set_queue", value="general_support") if observation.workspace.draft_reply is None: return TriageAction( action_type="draft_reply", value="Thanks for contacting support. We are reviewing this now.", ) return TriageAction(action_type="submit") workspace = observation.workspace if _normalize_choice(workspace.priority) != playbook["priority"]: return TriageAction(action_type="set_priority", value=playbook["priority"]) if _normalize_choice(workspace.queue) != playbook["queue"]: return TriageAction(action_type="set_queue", value=playbook["queue"]) if _normalize_choice(workspace.status) != playbook["status"]: return TriageAction(action_type="set_status", value=playbook["status"]) existing_tags = {_normalize_tag(tag) for tag in workspace.tags} for tag in playbook["tags"]: if _normalize_tag(tag) not in existing_tags: return TriageAction(action_type="add_tag", value=tag) if not _reply_covers_keywords(workspace.draft_reply, observation.reply_requirements): return TriageAction(action_type="draft_reply", value=playbook["reply"]) return TriageAction(action_type="submit") def _prefer_model_action(observation, candidate: TriageAction, planned: TriageAction) -> bool: if candidate.action_type != planned.action_type: return False if candidate.action_type in {"set_priority", "set_queue", "set_status"}: return _normalize_choice(candidate.value) == _normalize_choice(planned.value) if candidate.action_type in {"add_tag", "remove_tag"}: return _normalize_tag(candidate.value) == _normalize_tag(planned.value) if candidate.action_type == "draft_reply": return _reply_covers_keywords(candidate.value, observation.reply_requirements) return candidate.action_type == "submit" def _run_task(task_id: str) -> tuple[bool, list[float]]: env = SupportTriageEnvironment() rewards: list[float] = [] success = False score = 0.0 steps = 0 recent_steps: list[dict[str, str]] = [] print(f"[START] task={task_id} env=support_triage_env model={MODEL_NAME}", flush=True) try: observation = env.reset(task_id=task_id) while not observation.done: prompt = _build_prompt_with_history(observation, recent_steps) action = _planned_action(observation) try: candidate = _request_action_with_tools(prompt) except Exception: try: candidate = _request_action_with_json(prompt) except Exception: candidate = None if candidate is not None and _prefer_model_action(observation, candidate, action): action = candidate observation = env.step(action) steps += 1 rewards.append(float(observation.reward or 0.0)) error = _one_line(env.state.last_action_error) print( f"[STEP] step={steps} action={_action_for_log(action)} " f"reward={_format_reward(observation.reward)} done={str(observation.done).lower()} " f"error={error}", flush=True, ) recent_steps.append( { "action": _action_for_log(action), "reward": _format_reward(observation.reward), "error": error, } ) score = max(0.0, min(1.0, float(env.state.completion_score or 0.0))) success = env.state.submitted and score >= 0.85 return success, rewards finally: score = max(0.0, min(1.0, float(env.state.completion_score or 0.0))) env.close() joined_rewards = ",".join(_format_reward(value) for value in rewards) print( f"[END] success={str(success).lower()} steps={steps} score={score:.2f} rewards={joined_rewards}", flush=True, ) def main() -> None: selected_task_ids = task_ids() if TASK_IDS_OVERRIDE: selected_task_ids = [task_id.strip() for task_id in TASK_IDS_OVERRIDE.split(",") if task_id.strip()] if not selected_task_ids: raise ValueError("No task IDs selected. Set TASK_IDS to a comma-separated list or leave it unset.") for current_task_id in selected_task_ids: _run_task(current_task_id) if __name__ == "__main__": main()