Spaces:
Sleeping
Sleeping
Download inference.py from Neetu143/meta_hackathon: direct link, hf CLI and curl.
- Browser
- Download file 12.9 kB
-
https://huggingface.co/spaces/Neetu143/meta_hackathon/resolve/d754d1ee252db65e87480fefdf9b4ebe44ac6833/inference.py
- Command line
-
hf download hf://spaces/Neetu143/meta_hackathon@d754d1ee252db65e87480fefdf9b4ebe44ac6833/inference.py
-
curl -L -o inference.py https://huggingface.co/spaces/Neetu143/meta_hackathon/resolve/d754d1ee252db65e87480fefdf9b4ebe44ac6833/inference.py
12.9 kB
| from __future__ import annotations | |
| import json | |
| import os | |
| from openai import OpenAI | |
| from dotenv import load_dotenv | |
| from support_triage_env.models import TriageAction | |
| from support_triage_env.server.environment import SupportTriageEnvironment | |
| from support_triage_env.tasks import task_ids | |
| load_dotenv() | |
| API_BASE_URL = os.getenv("API_BASE_URL", "https://api.openai.com/v1") | |
| MODEL_NAME = os.getenv("MODEL_NAME", "gpt-4.1-mini") | |
| HF_TOKEN = os.getenv("HF_TOKEN") | |
| REQUEST_TIMEOUT_S = float(os.getenv("REQUEST_TIMEOUT_S", "60")) | |
| TASK_IDS_OVERRIDE = os.getenv("TASK_IDS") | |
| if HF_TOKEN is None: | |
| raise ValueError("HF_TOKEN environment variable is required") | |
| if HF_TOKEN.startswith("hf_") and API_BASE_URL == "https://api.openai.com/v1": | |
| raise ValueError( | |
| "HF_TOKEN looks like a Hugging Face Hub token, but API_BASE_URL points to OpenAI. " | |
| "Use a provider API key compatible with API_BASE_URL, or change API_BASE_URL to the " | |
| "endpoint that expects this token." | |
| ) | |
| client = OpenAI(base_url=API_BASE_URL, api_key=HF_TOKEN, timeout=REQUEST_TIMEOUT_S) | |
| ACTION_TOOL = [ | |
| { | |
| "type": "function", | |
| "function": { | |
| "name": "choose_action", | |
| "description": "Choose the next environment action.", | |
| "parameters": { | |
| "type": "object", | |
| "properties": { | |
| "action_type": { | |
| "type": "string", | |
| "enum": [ | |
| "set_priority", | |
| "set_queue", | |
| "set_status", | |
| "add_tag", | |
| "remove_tag", | |
| "draft_reply", | |
| "submit", | |
| ], | |
| }, | |
| "value": { | |
| "type": ["string", "null"], | |
| "description": "Action value, or null for submit.", | |
| }, | |
| }, | |
| "required": ["action_type", "value"], | |
| "additionalProperties": False, | |
| }, | |
| }, | |
| } | |
| ] | |
| TASK_PLAYBOOKS = { | |
| "billing_duplicate_charge": { | |
| "priority": "high", | |
| "queue": "billing", | |
| "status": "pending_refund_review", | |
| "tags": ["billing", "refund"], | |
| "reply": ( | |
| "I am sorry about the duplicate charge. We will process a refund for the duplicate " | |
| "charge within 24 hours while keeping your subscription active." | |
| ), | |
| }, | |
| "shipping_damaged_replacement": { | |
| "priority": "medium", | |
| "queue": "logistics", | |
| "status": "awaiting_verification", | |
| "tags": ["damaged", "replacement"], | |
| "reply": ( | |
| "I am sorry the monitor stand arrived damaged. We can process a replacement as soon " | |
| "as you send a photo of the damage and confirm your shipping address." | |
| ), | |
| }, | |
| "account_takeover_fraud": { | |
| "priority": "urgent", | |
| "queue": "trust_and_safety", | |
| "status": "escalated_security", | |
| "tags": ["security", "fraud", "account_takeover"], | |
| "reply": ( | |
| "We have escalated the unauthorized order incident and are working to secure the " | |
| "account. We will help you reset the password as part of this security response." | |
| ), | |
| }, | |
| } | |
| def _one_line(value: str | None) -> str: | |
| if value is None: | |
| return "null" | |
| return " ".join(value.replace("\n", " ").split()) | |
| def _format_reward(value: float | None) -> str: | |
| return f"{(value or 0.0):.2f}" | |
| def _action_for_log(action: TriageAction) -> str: | |
| if action.action_type == "submit": | |
| return "submit()" | |
| return f"{action.action_type}({json.dumps(action.value)})" | |
| def _build_prompt(observation) -> str: | |
| return _build_prompt_with_history(observation, []) | |
| def _build_prompt_with_history(observation, recent_steps: list[dict[str, str]]) -> str: | |
| payload = { | |
| "instructions": observation.instructions, | |
| "ticket": observation.ticket.model_dump(), | |
| "policy_notes": observation.policy_notes, | |
| "reply_requirements": observation.reply_requirements, | |
| "workspace": observation.workspace.model_dump(), | |
| "allowed_values": observation.allowed_values, | |
| "available_actions": observation.available_actions, | |
| "completion_score": observation.completion_score, | |
| "missing_items": observation.missing_items, | |
| "last_action_error": observation.last_action_error, | |
| "steps_remaining": observation.steps_remaining, | |
| "recent_steps": recent_steps[-4:], | |
| } | |
| solved_items = [key for key, score in observation.score_breakdown.items() if score >= 0.999] | |
| avoid_pairs = [ | |
| step["action"] | |
| for step in recent_steps[-4:] | |
| if step.get("reward", "").startswith("-") | |
| ] | |
| return ( | |
| "You are operating a customer support triage environment.\n" | |
| "Choose the single best next action.\n" | |
| "Use the choose_action function.\n" | |
| "Set value to null only for submit.\n" | |
| f"Already solved areas: {solved_items}.\n" | |
| f"Do not repeat recently bad actions: {avoid_pairs}.\n" | |
| "Prefer editing an unsolved area instead of revisiting a solved one.\n" | |
| f"Observation:{json.dumps(payload, separators=(',', ':'))}" | |
| ) | |
| def _extract_json_object(text: str) -> str: | |
| stripped = text.strip() | |
| if stripped.startswith("```"): | |
| stripped = stripped.strip("`") | |
| if stripped.startswith("json"): | |
| stripped = stripped[4:].strip() | |
| start = stripped.find("{") | |
| end = stripped.rfind("}") | |
| if start != -1 and end != -1 and end >= start: | |
| return stripped[start : end + 1] | |
| return stripped | |
| def _normalize_choice(value: str | None) -> str: | |
| return (value or "").strip().lower() | |
| def _normalize_tag(value: str | None) -> str: | |
| normalized = _normalize_choice(value).replace("-", "_").replace(" ", "_") | |
| while "__" in normalized: | |
| normalized = normalized.replace("__", "_") | |
| return normalized | |
| def _reply_covers_keywords(value: str | None, keywords: list[str]) -> bool: | |
| normalized = _normalize_choice(value).replace("_", " ").replace("-", " ") | |
| return all(_normalize_choice(keyword).replace("_", " ").replace("-", " ") in normalized for keyword in keywords) | |
| def _parse_action(content: str) -> TriageAction: | |
| data = json.loads(_extract_json_object(content)) | |
| return TriageAction( | |
| action_type=data["action_type"], | |
| value=data.get("value"), | |
| ) | |
| def _parse_tool_action(response) -> TriageAction | None: | |
| message = response.choices[0].message | |
| tool_calls = getattr(message, "tool_calls", None) or [] | |
| if not tool_calls: | |
| return None | |
| first_call = tool_calls[0] | |
| function = getattr(first_call, "function", None) | |
| if function is None or function.name != "choose_action": | |
| return None | |
| arguments = function.arguments or "{}" | |
| data = json.loads(arguments) | |
| return TriageAction( | |
| action_type=data["action_type"], | |
| value=data.get("value"), | |
| ) | |
| def _request_action_with_tools(prompt: str) -> TriageAction | None: | |
| response = client.chat.completions.create( | |
| model=MODEL_NAME, | |
| messages=[{"role": "user", "content": prompt}], | |
| temperature=0, | |
| tools=ACTION_TOOL, | |
| tool_choice="required", | |
| ) | |
| action = _parse_tool_action(response) | |
| if action is not None: | |
| return action | |
| content = response.choices[0].message.content or "" | |
| return _parse_action(content) | |
| def _request_action_with_json(prompt: str) -> TriageAction: | |
| response = client.chat.completions.create( | |
| model=MODEL_NAME, | |
| messages=[ | |
| { | |
| "role": "user", | |
| "content": ( | |
| f"{prompt}\n" | |
| "Return only compact JSON with keys action_type and value." | |
| ), | |
| } | |
| ], | |
| temperature=0, | |
| ) | |
| content = response.choices[0].message.content or "" | |
| return _parse_action(content) | |
| def _planned_action(observation) -> TriageAction: | |
| playbook = TASK_PLAYBOOKS.get(observation.task_id) | |
| if playbook is None: | |
| if observation.workspace.priority is None: | |
| return TriageAction(action_type="set_priority", value="medium") | |
| if observation.workspace.queue is None: | |
| return TriageAction(action_type="set_queue", value="general_support") | |
| if observation.workspace.draft_reply is None: | |
| return TriageAction( | |
| action_type="draft_reply", | |
| value="Thanks for contacting support. We are reviewing this now.", | |
| ) | |
| return TriageAction(action_type="submit") | |
| workspace = observation.workspace | |
| if _normalize_choice(workspace.priority) != playbook["priority"]: | |
| return TriageAction(action_type="set_priority", value=playbook["priority"]) | |
| if _normalize_choice(workspace.queue) != playbook["queue"]: | |
| return TriageAction(action_type="set_queue", value=playbook["queue"]) | |
| if _normalize_choice(workspace.status) != playbook["status"]: | |
| return TriageAction(action_type="set_status", value=playbook["status"]) | |
| existing_tags = {_normalize_tag(tag) for tag in workspace.tags} | |
| for tag in playbook["tags"]: | |
| if _normalize_tag(tag) not in existing_tags: | |
| return TriageAction(action_type="add_tag", value=tag) | |
| if not _reply_covers_keywords(workspace.draft_reply, observation.reply_requirements): | |
| return TriageAction(action_type="draft_reply", value=playbook["reply"]) | |
| return TriageAction(action_type="submit") | |
| def _prefer_model_action(observation, candidate: TriageAction, planned: TriageAction) -> bool: | |
| if candidate.action_type != planned.action_type: | |
| return False | |
| if candidate.action_type in {"set_priority", "set_queue", "set_status"}: | |
| return _normalize_choice(candidate.value) == _normalize_choice(planned.value) | |
| if candidate.action_type in {"add_tag", "remove_tag"}: | |
| return _normalize_tag(candidate.value) == _normalize_tag(planned.value) | |
| if candidate.action_type == "draft_reply": | |
| return _reply_covers_keywords(candidate.value, observation.reply_requirements) | |
| return candidate.action_type == "submit" | |
| def _run_task(task_id: str) -> tuple[bool, list[float]]: | |
| env = SupportTriageEnvironment() | |
| rewards: list[float] = [] | |
| success = False | |
| score = 0.0 | |
| steps = 0 | |
| recent_steps: list[dict[str, str]] = [] | |
| print(f"[START] task={task_id} env=support_triage_env model={MODEL_NAME}", flush=True) | |
| try: | |
| observation = env.reset(task_id=task_id) | |
| while not observation.done: | |
| prompt = _build_prompt_with_history(observation, recent_steps) | |
| action = _planned_action(observation) | |
| try: | |
| candidate = _request_action_with_tools(prompt) | |
| except Exception: | |
| try: | |
| candidate = _request_action_with_json(prompt) | |
| except Exception: | |
| candidate = None | |
| if candidate is not None and _prefer_model_action(observation, candidate, action): | |
| action = candidate | |
| observation = env.step(action) | |
| steps += 1 | |
| rewards.append(float(observation.reward or 0.0)) | |
| error = _one_line(env.state.last_action_error) | |
| print( | |
| f"[STEP] step={steps} action={_action_for_log(action)} " | |
| f"reward={_format_reward(observation.reward)} done={str(observation.done).lower()} " | |
| f"error={error}", | |
| flush=True, | |
| ) | |
| recent_steps.append( | |
| { | |
| "action": _action_for_log(action), | |
| "reward": _format_reward(observation.reward), | |
| "error": error, | |
| } | |
| ) | |
| score = max(0.0, min(1.0, float(env.state.completion_score or 0.0))) | |
| success = env.state.submitted and score >= 0.85 | |
| return success, rewards | |
| finally: | |
| score = max(0.0, min(1.0, float(env.state.completion_score or 0.0))) | |
| env.close() | |
| joined_rewards = ",".join(_format_reward(value) for value in rewards) | |
| print( | |
| f"[END] success={str(success).lower()} steps={steps} score={score:.2f} rewards={joined_rewards}", | |
| flush=True, | |
| ) | |
| def main() -> None: | |
| selected_task_ids = task_ids() | |
| if TASK_IDS_OVERRIDE: | |
| selected_task_ids = [task_id.strip() for task_id in TASK_IDS_OVERRIDE.split(",") if task_id.strip()] | |
| if not selected_task_ids: | |
| raise ValueError("No task IDs selected. Set TASK_IDS to a comma-separated list or leave it unset.") | |
| for current_task_id in selected_task_ids: | |
| _run_task(current_task_id) | |
| if __name__ == "__main__": | |
| main() | |