meta_hackathon / inference.py
Neetu143's picture
Upload support triage environment
0bbd62a verified
Raw History Blame Contribute Delete
19.6 kB
from __future__ import annotations
import json
import os
from typing import Any
from dotenv import load_dotenv
from openai import OpenAI
from support_triage_env.environment import SupportTriageCoreEnvironment
from support_triage_env.models import InvestigationRecord, TriageAction
from support_triage_env.tasks import task_ids
load_dotenv()
API_BASE_URL = os.getenv("API_BASE_URL", "https://api.openai.com/v1")
MODEL_NAME = os.getenv("MODEL_NAME", "gpt-4.1-mini")
HF_TOKEN = os.getenv("HF_TOKEN")
OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
API_KEY = OPENAI_API_KEY or HF_TOKEN
REQUEST_TIMEOUT_S = float(os.getenv("REQUEST_TIMEOUT_S", "60"))
TASK_IDS_OVERRIDE = os.getenv("TASK_IDS")
if API_KEY is None:
raise ValueError("OPENAI_API_KEY or HF_TOKEN environment variable is required")
if OPENAI_API_KEY is None and HF_TOKEN and HF_TOKEN.startswith("hf_") and API_BASE_URL == "https://api.openai.com/v1":
raise ValueError(
"HF_TOKEN looks like a Hugging Face Hub token, but API_BASE_URL points to OpenAI. "
"Use a provider API key compatible with API_BASE_URL, or change API_BASE_URL to the "
"endpoint that expects this token."
)
client = OpenAI(base_url=API_BASE_URL, api_key=API_KEY, timeout=REQUEST_TIMEOUT_S)
ACTION_TOOL = [
{
"type": "function",
"function": {
"name": "choose_action",
"description": "Choose the next environment action.",
"parameters": {
"type": "object",
"properties": {
"action_type": {
"type": "string",
"enum": [
"lookup_policy",
"lookup_order",
"lookup_account",
"lookup_fraud_signal",
"set_priority",
"set_queue",
"set_status",
"add_tag",
"remove_tag",
"add_internal_note",
"draft_reply",
"submit",
],
},
"value": {
"type": ["string", "null"],
"description": "Action value, or null for submit.",
},
},
"required": ["action_type", "value"],
"additionalProperties": False,
},
},
}
]
def _one_line(value: str | None) -> str:
if value is None:
return "null"
return " ".join(value.replace("\n", " ").split())
def _format_reward(value: float | None) -> str:
return f"{(value or 0.0):.2f}"
def _normalize_choice(value: str | None) -> str:
return " ".join((value or "").strip().lower().split())
def _normalize_text(value: str | None) -> str:
return _normalize_choice(value).replace("_", " ").replace("-", " ")
def _normalize_tag(value: str | None) -> str:
normalized = _normalize_text(value).replace(" ", "_")
while "__" in normalized:
normalized = normalized.replace("__", "_")
return normalized
def _matches_expected_text(value: str | None, expected: str) -> bool:
return _normalize_text(value) == _normalize_text(expected)
def _action_for_log(action: TriageAction) -> str:
if action.action_type == "submit":
return "submit()"
return f"{action.action_type}({json.dumps(action.value)})"
def _record_index(observation) -> dict[tuple[str, str], InvestigationRecord]:
return {
(record.source, _normalize_choice(record.target)): record
for record in observation.investigation_log
}
def _record_for(observation, source: str, target: str | None) -> InvestigationRecord | None:
return _record_index(observation).get((source, _normalize_choice(target)))
def _lookup_value(record: InvestigationRecord | None, key: str, default: str = "") -> str:
if record is None:
return default
return record.key_fields.get(key, default)
def _build_prompt_with_history(observation, recent_steps: list[dict[str, str]]) -> str:
payload = {
"case_title": observation.case_title,
"scenario_summary": observation.scenario_summary,
"instructions": observation.instructions,
"ticket": observation.ticket.model_dump(),
"policy_notes": observation.policy_notes,
"investigation_requirements": observation.investigation_requirements,
"investigation_log": [record.model_dump() for record in observation.investigation_log],
"workspace": observation.workspace.model_dump(),
"allowed_values": observation.allowed_values,
"available_actions": observation.available_actions,
"last_action_error": observation.last_action_error,
"steps_remaining": observation.steps_remaining,
"recent_steps": recent_steps[-5:],
}
avoid_pairs = [
step["action"]
for step in recent_steps[-5:]
if step.get("reward", "").startswith("-")
]
return (
"You are operating a customer-support operations environment.\n"
"Look up missing evidence before finalizing the workflow.\n"
"Choose exactly one next action with the choose_action function.\n"
"Set value to null only for submit.\n"
f"Do not repeat recently bad actions: {avoid_pairs}.\n"
"Prefer unresolved investigation and policy-backed workflow decisions over guesswork.\n"
f"Observation:{json.dumps(payload, separators=(',', ':'))}"
)
def _extract_json_object(text: str) -> str:
stripped = text.strip()
if stripped.startswith("```"):
stripped = stripped.strip("`")
if stripped.startswith("json"):
stripped = stripped[4:].strip()
start = stripped.find("{")
end = stripped.rfind("}")
if start != -1 and end != -1 and end >= start:
return stripped[start : end + 1]
return stripped
def _parse_action(content: str) -> TriageAction:
data = json.loads(_extract_json_object(content))
return TriageAction(action_type=data["action_type"], value=data.get("value"))
def _parse_tool_action(response) -> TriageAction | None:
message = response.choices[0].message
tool_calls = getattr(message, "tool_calls", None) or []
if not tool_calls:
return None
first_call = tool_calls[0]
function = getattr(first_call, "function", None)
if function is None or function.name != "choose_action":
return None
data = json.loads(function.arguments or "{}")
return TriageAction(action_type=data["action_type"], value=data.get("value"))
def _request_action_with_tools(prompt: str) -> TriageAction | None:
response = client.chat.completions.create(
model=MODEL_NAME,
messages=[{"role": "user", "content": prompt}],
temperature=0,
tools=ACTION_TOOL,
tool_choice="required",
)
action = _parse_tool_action(response)
if action is not None:
return action
content = response.choices[0].message.content or ""
return _parse_action(content)
def _request_action_with_json(prompt: str) -> TriageAction:
response = client.chat.completions.create(
model=MODEL_NAME,
messages=[
{
"role": "user",
"content": (
f"{prompt}\n"
"Return only compact JSON with keys action_type and value."
),
}
],
temperature=0,
)
content = response.choices[0].message.content or ""
return _parse_action(content)
def _required_lookup_plan(observation) -> list[tuple[str, str]]:
if observation.task_id == "billing_duplicate_charge":
return [
("lookup_policy", "billing_charge_anomaly"),
("lookup_order", observation.ticket.order_id or ""),
]
if observation.task_id == "shipping_damaged_replacement":
return [
("lookup_policy", "damage_replacement"),
("lookup_order", observation.ticket.order_id or ""),
]
if observation.task_id == "account_takeover_fraud":
return [
("lookup_policy", "account_takeover"),
("lookup_account", observation.ticket.account_email),
("lookup_order", observation.ticket.order_id or ""),
("lookup_fraud_signal", observation.ticket.ticket_id),
]
return []
def _billing_resolution(observation) -> dict[str, Any]:
order_record = _record_for(observation, "order", observation.ticket.order_id)
charge_pattern = _lookup_value(order_record, "charge_pattern", "duplicate_capture")
tier = observation.ticket.customer_tier
if charge_pattern == "authorization_hold":
return {
"priority": "medium" if tier == "standard" else "high",
"queue": "billing",
"status": "resolved",
"tags": ["billing", "auth_hold"],
"note": (
"The ledger shows an authorization hold and only one settled charge. "
"No refund workflow is needed because the hold release should happen automatically."
),
"reply": (
"I reviewed the billing activity and found only one settled charge. "
"The second line item is an authorization hold, not a second captured payment, "
"so no refund workflow is needed. The hold should disappear within 3-5 business days."
),
}
return {
"priority": "high" if tier in {"plus", "enterprise"} else "medium",
"queue": "billing",
"status": "pending_refund_review",
"tags": ["billing", "refund", "duplicate_charge"],
"note": (
"The ledger shows two settled charges for the same renewal. "
"Route the case into refund review while the subscription remains active."
),
"reply": (
"I am sorry about the duplicate charge. I reviewed the renewal and confirmed the extra charge. "
"We are moving this into refund review now, and you will receive an update within 24 hours. "
"Your subscription remains active."
),
}
def _shipping_resolution(observation) -> dict[str, Any]:
order_record = _record_for(observation, "order", observation.ticket.order_id)
damage_evidence = _lookup_value(order_record, "damage_evidence", "missing")
if damage_evidence == "verified":
return {
"priority": "high",
"queue": "logistics",
"status": "open",
"tags": ["damaged", "replacement", "carrier_confirmed"],
"note": (
"Carrier-confirmed damage is already documented and the damage photos are already on file. "
"Logistics can dispatch the replacement using the verified address."
),
"reply": (
"We already have the damage photos on file, and the replacement is approved. "
"We will use the verified address on file and move the replacement into dispatch."
),
}
return {
"priority": "medium",
"queue": "logistics",
"status": "awaiting_verification",
"tags": ["damaged", "replacement"],
"note": (
"Damage evidence is missing, so a photo is required before dispatch. "
"We also still need address confirmation."
),
"reply": (
"We can process the replacement for you. "
"Please send a photo of the damage and confirm your shipping address so logistics can proceed."
),
}
def _security_resolution(observation) -> dict[str, Any]:
account_record = _record_for(observation, "account", observation.ticket.account_email)
order_record = _record_for(observation, "order", observation.ticket.order_id)
fraud_record = _record_for(observation, "fraud", observation.ticket.ticket_id)
access_state = _lookup_value(account_record, "access_state", "locked_out")
fraud_score = float(_lookup_value(fraud_record, "fraud_score", "0.95") or "0.95")
order_type = _lookup_value(order_record, "unauthorized_order", "submitted")
priority = "urgent" if access_state == "locked_out" or fraud_score >= 0.95 else "high"
tags = ["security", "account_takeover", "fraud", "unauthorized_order"]
if order_type == "blocked_attempt":
return {
"priority": priority,
"queue": "trust_and_safety",
"status": "escalated_security",
"tags": tags,
"note": (
"The account email changed, a blocked order attempt is linked to the incident, "
"and the customer is locked out. Keep the case in escalated security."
),
"reply": (
"This incident is escalated to our trust and safety team so we can secure the account. "
"The blocked order attempt is linked to the takeover, and we will help you complete a password reset as part of the recovery process."
),
}
return {
"priority": priority,
"queue": "trust_and_safety",
"status": "escalated_security",
"tags": tags,
"note": (
"The account email changed, an unauthorized order is linked to the incident, "
"and the customer is locked out. Escalate this for security review immediately."
),
"reply": (
"This incident is escalated to our trust and safety team so we can secure the account. "
"The unauthorized order is linked to the takeover, and we will help you complete a password reset as part of the recovery process."
),
}
def _resolution_plan(observation) -> dict[str, Any]:
if observation.task_id == "billing_duplicate_charge":
return _billing_resolution(observation)
if observation.task_id == "shipping_damaged_replacement":
return _shipping_resolution(observation)
if observation.task_id == "account_takeover_fraud":
return _security_resolution(observation)
return {
"priority": "medium",
"queue": "general_support",
"status": "open",
"tags": [],
"note": "Initial investigation completed.",
"reply": "Thanks for contacting support. We are reviewing this now.",
}
def _planned_action(observation) -> TriageAction:
observed_lookups = _record_index(observation)
for action_type, target in _required_lookup_plan(observation):
source = action_type.replace("lookup_", "").replace("fraud_signal", "fraud")
if (source, _normalize_choice(target)) not in observed_lookups:
return TriageAction(action_type=action_type, value=target)
plan = _resolution_plan(observation)
workspace = observation.workspace
if _normalize_choice(workspace.priority) != _normalize_choice(plan["priority"]):
return TriageAction(action_type="set_priority", value=plan["priority"])
if _normalize_choice(workspace.queue) != _normalize_choice(plan["queue"]):
return TriageAction(action_type="set_queue", value=plan["queue"])
if _normalize_choice(workspace.status) != _normalize_choice(plan["status"]):
return TriageAction(action_type="set_status", value=plan["status"])
existing_tags = {_normalize_tag(tag) for tag in workspace.tags}
for tag in plan["tags"]:
if _normalize_tag(tag) not in existing_tags:
return TriageAction(action_type="add_tag", value=tag)
if not _matches_expected_text(workspace.internal_note, plan["note"]):
return TriageAction(action_type="add_internal_note", value=plan["note"])
if not _matches_expected_text(workspace.draft_reply, plan["reply"]):
return TriageAction(action_type="draft_reply", value=plan["reply"])
return TriageAction(action_type="submit")
def _prefer_model_action(observation, candidate: TriageAction, planned: TriageAction) -> bool:
if candidate.action_type != planned.action_type:
return False
if candidate.action_type in {
"lookup_policy",
"lookup_order",
"lookup_account",
"lookup_fraud_signal",
"set_priority",
"set_queue",
"set_status",
}:
return _normalize_choice(candidate.value) == _normalize_choice(planned.value)
if candidate.action_type in {"add_tag", "remove_tag"}:
return _normalize_tag(candidate.value) == _normalize_tag(planned.value)
if candidate.action_type == "add_internal_note":
return _matches_expected_text(candidate.value, planned.value or "")
if candidate.action_type == "draft_reply":
return _matches_expected_text(candidate.value, planned.value or "")
return candidate.action_type == "submit"
def _run_task(task_id: str) -> tuple[bool, list[float]]:
env = SupportTriageCoreEnvironment()
rewards: list[float] = []
success = False
score = 0.0
steps = 0
recent_steps: list[dict[str, str]] = []
print(f"[START] task={task_id} env=support_triage_env model={MODEL_NAME}", flush=True)
try:
observation = env.reset(task_id=task_id)
state = env.state()
while not observation.done:
prompt = _build_prompt_with_history(observation, recent_steps)
action = _planned_action(observation)
try:
candidate = _request_action_with_tools(prompt)
except Exception:
try:
candidate = _request_action_with_json(prompt)
except Exception:
candidate = None
if candidate is not None and _prefer_model_action(observation, candidate, action):
action = candidate
result = env.step(action)
observation = result.observation
steps += 1
rewards.append(float(result.reward.value))
state = env.state()
error = _one_line(result.info.last_action_error)
print(
f"[STEP] step={steps} action={_action_for_log(action)} "
f"reward={_format_reward(result.reward.value)} done={str(result.done).lower()} "
f"error={error}",
flush=True,
)
recent_steps.append(
{
"action": _action_for_log(action),
"reward": _format_reward(result.reward.value),
"error": error,
}
)
score = max(0.0, min(1.0, float(state.completion_score or 0.0)))
success = state.submitted and score >= 0.90
return success, rewards
finally:
final_state = env.state()
score = max(0.0, min(1.0, float(final_state.completion_score or 0.0)))
env.close()
joined_rewards = ",".join(_format_reward(value) for value in rewards)
print(
f"[END] success={str(success).lower()} steps={steps} score={score:.2f} rewards={joined_rewards}",
flush=True,
)
def main() -> None:
selected_task_ids = task_ids()
if TASK_IDS_OVERRIDE:
selected_task_ids = [task_id.strip() for task_id in TASK_IDS_OVERRIDE.split(",") if task_id.strip()]
if not selected_task_ids:
raise ValueError("No task IDs selected. Set TASK_IDS to a comma-separated list or leave it unset.")
for current_task_id in selected_task_ids:
_run_task(current_task_id)
if __name__ == "__main__":
main()