meta_hackathon / inference.py
Neetu143's picture
Upload support triage environment
d754d1e verified
Raw History Blame
12.9 kB
from __future__ import annotations
import json
import os
from openai import OpenAI
from dotenv import load_dotenv
from support_triage_env.models import TriageAction
from support_triage_env.server.environment import SupportTriageEnvironment
from support_triage_env.tasks import task_ids
load_dotenv()
API_BASE_URL = os.getenv("API_BASE_URL", "https://api.openai.com/v1")
MODEL_NAME = os.getenv("MODEL_NAME", "gpt-4.1-mini")
HF_TOKEN = os.getenv("HF_TOKEN")
REQUEST_TIMEOUT_S = float(os.getenv("REQUEST_TIMEOUT_S", "60"))
TASK_IDS_OVERRIDE = os.getenv("TASK_IDS")
if HF_TOKEN is None:
raise ValueError("HF_TOKEN environment variable is required")
if HF_TOKEN.startswith("hf_") and API_BASE_URL == "https://api.openai.com/v1":
raise ValueError(
"HF_TOKEN looks like a Hugging Face Hub token, but API_BASE_URL points to OpenAI. "
"Use a provider API key compatible with API_BASE_URL, or change API_BASE_URL to the "
"endpoint that expects this token."
)
client = OpenAI(base_url=API_BASE_URL, api_key=HF_TOKEN, timeout=REQUEST_TIMEOUT_S)
ACTION_TOOL = [
{
"type": "function",
"function": {
"name": "choose_action",
"description": "Choose the next environment action.",
"parameters": {
"type": "object",
"properties": {
"action_type": {
"type": "string",
"enum": [
"set_priority",
"set_queue",
"set_status",
"add_tag",
"remove_tag",
"draft_reply",
"submit",
],
},
"value": {
"type": ["string", "null"],
"description": "Action value, or null for submit.",
},
},
"required": ["action_type", "value"],
"additionalProperties": False,
},
},
}
]
TASK_PLAYBOOKS = {
"billing_duplicate_charge": {
"priority": "high",
"queue": "billing",
"status": "pending_refund_review",
"tags": ["billing", "refund"],
"reply": (
"I am sorry about the duplicate charge. We will process a refund for the duplicate "
"charge within 24 hours while keeping your subscription active."
),
},
"shipping_damaged_replacement": {
"priority": "medium",
"queue": "logistics",
"status": "awaiting_verification",
"tags": ["damaged", "replacement"],
"reply": (
"I am sorry the monitor stand arrived damaged. We can process a replacement as soon "
"as you send a photo of the damage and confirm your shipping address."
),
},
"account_takeover_fraud": {
"priority": "urgent",
"queue": "trust_and_safety",
"status": "escalated_security",
"tags": ["security", "fraud", "account_takeover"],
"reply": (
"We have escalated the unauthorized order incident and are working to secure the "
"account. We will help you reset the password as part of this security response."
),
},
}
def _one_line(value: str | None) -> str:
if value is None:
return "null"
return " ".join(value.replace("\n", " ").split())
def _format_reward(value: float | None) -> str:
return f"{(value or 0.0):.2f}"
def _action_for_log(action: TriageAction) -> str:
if action.action_type == "submit":
return "submit()"
return f"{action.action_type}({json.dumps(action.value)})"
def _build_prompt(observation) -> str:
return _build_prompt_with_history(observation, [])
def _build_prompt_with_history(observation, recent_steps: list[dict[str, str]]) -> str:
payload = {
"instructions": observation.instructions,
"ticket": observation.ticket.model_dump(),
"policy_notes": observation.policy_notes,
"reply_requirements": observation.reply_requirements,
"workspace": observation.workspace.model_dump(),
"allowed_values": observation.allowed_values,
"available_actions": observation.available_actions,
"completion_score": observation.completion_score,
"missing_items": observation.missing_items,
"last_action_error": observation.last_action_error,
"steps_remaining": observation.steps_remaining,
"recent_steps": recent_steps[-4:],
}
solved_items = [key for key, score in observation.score_breakdown.items() if score >= 0.999]
avoid_pairs = [
step["action"]
for step in recent_steps[-4:]
if step.get("reward", "").startswith("-")
]
return (
"You are operating a customer support triage environment.\n"
"Choose the single best next action.\n"
"Use the choose_action function.\n"
"Set value to null only for submit.\n"
f"Already solved areas: {solved_items}.\n"
f"Do not repeat recently bad actions: {avoid_pairs}.\n"
"Prefer editing an unsolved area instead of revisiting a solved one.\n"
f"Observation:{json.dumps(payload, separators=(',', ':'))}"
)
def _extract_json_object(text: str) -> str:
stripped = text.strip()
if stripped.startswith("```"):
stripped = stripped.strip("`")
if stripped.startswith("json"):
stripped = stripped[4:].strip()
start = stripped.find("{")
end = stripped.rfind("}")
if start != -1 and end != -1 and end >= start:
return stripped[start : end + 1]
return stripped
def _normalize_choice(value: str | None) -> str:
return (value or "").strip().lower()
def _normalize_tag(value: str | None) -> str:
normalized = _normalize_choice(value).replace("-", "_").replace(" ", "_")
while "__" in normalized:
normalized = normalized.replace("__", "_")
return normalized
def _reply_covers_keywords(value: str | None, keywords: list[str]) -> bool:
normalized = _normalize_choice(value).replace("_", " ").replace("-", " ")
return all(_normalize_choice(keyword).replace("_", " ").replace("-", " ") in normalized for keyword in keywords)
def _parse_action(content: str) -> TriageAction:
data = json.loads(_extract_json_object(content))
return TriageAction(
action_type=data["action_type"],
value=data.get("value"),
)
def _parse_tool_action(response) -> TriageAction | None:
message = response.choices[0].message
tool_calls = getattr(message, "tool_calls", None) or []
if not tool_calls:
return None
first_call = tool_calls[0]
function = getattr(first_call, "function", None)
if function is None or function.name != "choose_action":
return None
arguments = function.arguments or "{}"
data = json.loads(arguments)
return TriageAction(
action_type=data["action_type"],
value=data.get("value"),
)
def _request_action_with_tools(prompt: str) -> TriageAction | None:
response = client.chat.completions.create(
model=MODEL_NAME,
messages=[{"role": "user", "content": prompt}],
temperature=0,
tools=ACTION_TOOL,
tool_choice="required",
)
action = _parse_tool_action(response)
if action is not None:
return action
content = response.choices[0].message.content or ""
return _parse_action(content)
def _request_action_with_json(prompt: str) -> TriageAction:
response = client.chat.completions.create(
model=MODEL_NAME,
messages=[
{
"role": "user",
"content": (
f"{prompt}\n"
"Return only compact JSON with keys action_type and value."
),
}
],
temperature=0,
)
content = response.choices[0].message.content or ""
return _parse_action(content)
def _planned_action(observation) -> TriageAction:
playbook = TASK_PLAYBOOKS.get(observation.task_id)
if playbook is None:
if observation.workspace.priority is None:
return TriageAction(action_type="set_priority", value="medium")
if observation.workspace.queue is None:
return TriageAction(action_type="set_queue", value="general_support")
if observation.workspace.draft_reply is None:
return TriageAction(
action_type="draft_reply",
value="Thanks for contacting support. We are reviewing this now.",
)
return TriageAction(action_type="submit")
workspace = observation.workspace
if _normalize_choice(workspace.priority) != playbook["priority"]:
return TriageAction(action_type="set_priority", value=playbook["priority"])
if _normalize_choice(workspace.queue) != playbook["queue"]:
return TriageAction(action_type="set_queue", value=playbook["queue"])
if _normalize_choice(workspace.status) != playbook["status"]:
return TriageAction(action_type="set_status", value=playbook["status"])
existing_tags = {_normalize_tag(tag) for tag in workspace.tags}
for tag in playbook["tags"]:
if _normalize_tag(tag) not in existing_tags:
return TriageAction(action_type="add_tag", value=tag)
if not _reply_covers_keywords(workspace.draft_reply, observation.reply_requirements):
return TriageAction(action_type="draft_reply", value=playbook["reply"])
return TriageAction(action_type="submit")
def _prefer_model_action(observation, candidate: TriageAction, planned: TriageAction) -> bool:
if candidate.action_type != planned.action_type:
return False
if candidate.action_type in {"set_priority", "set_queue", "set_status"}:
return _normalize_choice(candidate.value) == _normalize_choice(planned.value)
if candidate.action_type in {"add_tag", "remove_tag"}:
return _normalize_tag(candidate.value) == _normalize_tag(planned.value)
if candidate.action_type == "draft_reply":
return _reply_covers_keywords(candidate.value, observation.reply_requirements)
return candidate.action_type == "submit"
def _run_task(task_id: str) -> tuple[bool, list[float]]:
env = SupportTriageEnvironment()
rewards: list[float] = []
success = False
score = 0.0
steps = 0
recent_steps: list[dict[str, str]] = []
print(f"[START] task={task_id} env=support_triage_env model={MODEL_NAME}", flush=True)
try:
observation = env.reset(task_id=task_id)
while not observation.done:
prompt = _build_prompt_with_history(observation, recent_steps)
action = _planned_action(observation)
try:
candidate = _request_action_with_tools(prompt)
except Exception:
try:
candidate = _request_action_with_json(prompt)
except Exception:
candidate = None
if candidate is not None and _prefer_model_action(observation, candidate, action):
action = candidate
observation = env.step(action)
steps += 1
rewards.append(float(observation.reward or 0.0))
error = _one_line(env.state.last_action_error)
print(
f"[STEP] step={steps} action={_action_for_log(action)} "
f"reward={_format_reward(observation.reward)} done={str(observation.done).lower()} "
f"error={error}",
flush=True,
)
recent_steps.append(
{
"action": _action_for_log(action),
"reward": _format_reward(observation.reward),
"error": error,
}
)
score = max(0.0, min(1.0, float(env.state.completion_score or 0.0)))
success = env.state.submitted and score >= 0.85
return success, rewards
finally:
score = max(0.0, min(1.0, float(env.state.completion_score or 0.0)))
env.close()
joined_rewards = ",".join(_format_reward(value) for value in rewards)
print(
f"[END] success={str(success).lower()} steps={steps} score={score:.2f} rewards={joined_rewards}",
flush=True,
)
def main() -> None:
selected_task_ids = task_ids()
if TASK_IDS_OVERRIDE:
selected_task_ids = [task_id.strip() for task_id in TASK_IDS_OVERRIDE.split(",") if task_id.strip()]
if not selected_task_ids:
raise ValueError("No task IDs selected. Set TASK_IDS to a comma-separated list or leave it unset.")
for current_task_id in selected_task_ids:
_run_task(current_task_id)
if __name__ == "__main__":
main()