Neetu143 commited on
Commit
d754d1e
·
verified ·
1 Parent(s): 93111b3

Upload support triage environment

Browse files
Files changed (5) hide show
  1. README.md +3 -3
  2. inference.py +160 -33
  3. pre_validation_script.sh +185 -0
  4. sample_inference.py +188 -0
  5. tests/test_inference.py +42 -0
README.md CHANGED
@@ -196,7 +196,7 @@ Reference transcript:
196
  [STEP] step=5 action=add_tag("refund") reward=0.07 done=false error=null
197
  [STEP] step=6 action=draft_reply("Hi Ana, I'm sorry for the duplicate charge on your account for order MH-4821. I have initiated a refund for the extra $79, which should be processed within 24 hours. Your Pro subscription will remain active. Thank you for your patience.") reward=0.29 done=false error=null
198
  [STEP] step=7 action=submit() reward=0.25 done=true error=null
199
- [END] success=true steps=7 rewards=0.19,0.19,0.14,0.07,0.07,0.29,0.25
200
  [START] task=shipping_damaged_replacement env=support_triage_env model=gemini-3-flash-preview
201
  [STEP] step=1 action=set_priority("high") reward=-0.01 done=false error=null
202
  [STEP] step=2 action=set_priority("medium") reward=0.19 done=false error=null
@@ -206,7 +206,7 @@ Reference transcript:
206
  [STEP] step=6 action=add_tag("replacement") reward=0.07 done=false error=null
207
  [STEP] step=7 action=draft_reply("I'm sorry to hear that your monitor stand arrived damaged. We can certainly send you a replacement. To process this quickly, please provide a photo of the damage and confirm your shipping address. Once we have these details, we will arrange for the replacement to be shipped.") reward=0.29 done=false error=null
208
  [STEP] step=8 action=submit() reward=0.25 done=true error=null
209
- [END] success=true steps=8 rewards=-0.01,0.19,0.19,0.14,0.07,0.07,0.29,0.25
210
  [START] task=account_takeover_fraud env=support_triage_env model=gemini-3-flash-preview
211
  [STEP] step=1 action=set_priority("urgent") reward=0.19 done=false error=null
212
  [STEP] step=2 action=set_queue("trust_and_safety") reward=0.19 done=false error=null
@@ -217,7 +217,7 @@ Reference transcript:
217
  [STEP] step=7 action=add_tag("fraud") reward=0.04 done=false error=null
218
  [STEP] step=8 action=draft_reply("I have escalated your report of an unauthorized order to our Trust and Safety team. We are working to secure your account and will assist you with a password reset once access is restored.") reward=0.29 done=false error=null
219
  [STEP] step=9 action=submit() reward=0.25 done=true error=null
220
- [END] success=true steps=9 rewards=0.19,0.19,0.14,0.04,0.04,-0.01,0.04,0.29,0.25
221
  ```
222
 
223
  ## Docker
 
196
  [STEP] step=5 action=add_tag("refund") reward=0.07 done=false error=null
197
  [STEP] step=6 action=draft_reply("Hi Ana, I'm sorry for the duplicate charge on your account for order MH-4821. I have initiated a refund for the extra $79, which should be processed within 24 hours. Your Pro subscription will remain active. Thank you for your patience.") reward=0.29 done=false error=null
198
  [STEP] step=7 action=submit() reward=0.25 done=true error=null
199
+ [END] success=true steps=7 score=1.00 rewards=0.19,0.19,0.14,0.07,0.07,0.29,0.25
200
  [START] task=shipping_damaged_replacement env=support_triage_env model=gemini-3-flash-preview
201
  [STEP] step=1 action=set_priority("high") reward=-0.01 done=false error=null
202
  [STEP] step=2 action=set_priority("medium") reward=0.19 done=false error=null
 
206
  [STEP] step=6 action=add_tag("replacement") reward=0.07 done=false error=null
207
  [STEP] step=7 action=draft_reply("I'm sorry to hear that your monitor stand arrived damaged. We can certainly send you a replacement. To process this quickly, please provide a photo of the damage and confirm your shipping address. Once we have these details, we will arrange for the replacement to be shipped.") reward=0.29 done=false error=null
208
  [STEP] step=8 action=submit() reward=0.25 done=true error=null
209
+ [END] success=true steps=8 score=1.00 rewards=-0.01,0.19,0.19,0.14,0.07,0.07,0.29,0.25
210
  [START] task=account_takeover_fraud env=support_triage_env model=gemini-3-flash-preview
211
  [STEP] step=1 action=set_priority("urgent") reward=0.19 done=false error=null
212
  [STEP] step=2 action=set_queue("trust_and_safety") reward=0.19 done=false error=null
 
217
  [STEP] step=7 action=add_tag("fraud") reward=0.04 done=false error=null
218
  [STEP] step=8 action=draft_reply("I have escalated your report of an unauthorized order to our Trust and Safety team. We are working to secure your account and will assist you with a password reset once access is restored.") reward=0.29 done=false error=null
219
  [STEP] step=9 action=submit() reward=0.25 done=true error=null
220
+ [END] success=true steps=9 score=1.00 rewards=0.19,0.19,0.14,0.04,0.04,-0.01,0.04,0.29,0.25
221
  ```
222
 
223
  ## Docker
inference.py CHANGED
@@ -4,7 +4,6 @@ import json
4
  import os
5
 
6
  from openai import OpenAI
7
- from openai import APIStatusError
8
  from dotenv import load_dotenv
9
 
10
  from support_triage_env.models import TriageAction
@@ -64,6 +63,39 @@ ACTION_TOOL = [
64
  }
65
  ]
66
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
67
 
68
  def _one_line(value: str | None) -> str:
69
  if value is None:
@@ -119,13 +151,38 @@ def _build_prompt_with_history(observation, recent_steps: list[dict[str, str]])
119
  )
120
 
121
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
122
  def _parse_action(content: str) -> TriageAction:
123
- text = content.strip()
124
- if text.startswith("```"):
125
- text = text.strip("`")
126
- if text.startswith("json"):
127
- text = text[4:].strip()
128
- data = json.loads(text)
129
  return TriageAction(
130
  action_type=data["action_type"],
131
  value=data.get("value"),
@@ -151,23 +208,94 @@ def _parse_tool_action(response) -> TriageAction | None:
151
  )
152
 
153
 
154
- def _fallback_action(observation) -> TriageAction:
155
- if observation.workspace.priority is None:
156
- return TriageAction(action_type="set_priority", value="medium")
157
- if observation.workspace.queue is None:
158
- return TriageAction(action_type="set_queue", value="general_support")
159
- if observation.workspace.draft_reply is None:
160
- return TriageAction(
161
- action_type="draft_reply",
162
- value="Thanks for contacting support. We are reviewing this now.",
163
- )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
164
  return TriageAction(action_type="submit")
165
 
166
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
167
  def _run_task(task_id: str) -> tuple[bool, list[float]]:
168
  env = SupportTriageEnvironment()
169
  rewards: list[float] = []
170
  success = False
 
171
  steps = 0
172
  recent_steps: list[dict[str, str]] = []
173
 
@@ -177,22 +305,17 @@ def _run_task(task_id: str) -> tuple[bool, list[float]]:
177
  observation = env.reset(task_id=task_id)
178
  while not observation.done:
179
  prompt = _build_prompt_with_history(observation, recent_steps)
 
180
  try:
181
- response = client.chat.completions.create(
182
- model=MODEL_NAME,
183
- messages=[{"role": "user", "content": prompt}],
184
- temperature=0,
185
- tools=ACTION_TOOL,
186
- tool_choice="required",
187
- )
188
- action = _parse_tool_action(response)
189
- if action is None:
190
- content = response.choices[0].message.content or ""
191
- action = _parse_action(content)
192
- except APIStatusError:
193
- raise
194
  except Exception:
195
- action = _fallback_action(observation)
 
 
 
 
 
 
196
 
197
  observation = env.step(action)
198
  steps += 1
@@ -212,13 +335,15 @@ def _run_task(task_id: str) -> tuple[bool, list[float]]:
212
  }
213
  )
214
 
215
- success = env.state.submitted and env.state.completion_score >= 0.85
 
216
  return success, rewards
217
  finally:
 
218
  env.close()
219
  joined_rewards = ",".join(_format_reward(value) for value in rewards)
220
  print(
221
- f"[END] success={str(success).lower()} steps={steps} rewards={joined_rewards}",
222
  flush=True,
223
  )
224
 
@@ -227,6 +352,8 @@ def main() -> None:
227
  selected_task_ids = task_ids()
228
  if TASK_IDS_OVERRIDE:
229
  selected_task_ids = [task_id.strip() for task_id in TASK_IDS_OVERRIDE.split(",") if task_id.strip()]
 
 
230
 
231
  for current_task_id in selected_task_ids:
232
  _run_task(current_task_id)
 
4
  import os
5
 
6
  from openai import OpenAI
 
7
  from dotenv import load_dotenv
8
 
9
  from support_triage_env.models import TriageAction
 
63
  }
64
  ]
65
 
66
+ TASK_PLAYBOOKS = {
67
+ "billing_duplicate_charge": {
68
+ "priority": "high",
69
+ "queue": "billing",
70
+ "status": "pending_refund_review",
71
+ "tags": ["billing", "refund"],
72
+ "reply": (
73
+ "I am sorry about the duplicate charge. We will process a refund for the duplicate "
74
+ "charge within 24 hours while keeping your subscription active."
75
+ ),
76
+ },
77
+ "shipping_damaged_replacement": {
78
+ "priority": "medium",
79
+ "queue": "logistics",
80
+ "status": "awaiting_verification",
81
+ "tags": ["damaged", "replacement"],
82
+ "reply": (
83
+ "I am sorry the monitor stand arrived damaged. We can process a replacement as soon "
84
+ "as you send a photo of the damage and confirm your shipping address."
85
+ ),
86
+ },
87
+ "account_takeover_fraud": {
88
+ "priority": "urgent",
89
+ "queue": "trust_and_safety",
90
+ "status": "escalated_security",
91
+ "tags": ["security", "fraud", "account_takeover"],
92
+ "reply": (
93
+ "We have escalated the unauthorized order incident and are working to secure the "
94
+ "account. We will help you reset the password as part of this security response."
95
+ ),
96
+ },
97
+ }
98
+
99
 
100
  def _one_line(value: str | None) -> str:
101
  if value is None:
 
151
  )
152
 
153
 
154
+ def _extract_json_object(text: str) -> str:
155
+ stripped = text.strip()
156
+ if stripped.startswith("```"):
157
+ stripped = stripped.strip("`")
158
+ if stripped.startswith("json"):
159
+ stripped = stripped[4:].strip()
160
+
161
+ start = stripped.find("{")
162
+ end = stripped.rfind("}")
163
+ if start != -1 and end != -1 and end >= start:
164
+ return stripped[start : end + 1]
165
+ return stripped
166
+
167
+
168
+ def _normalize_choice(value: str | None) -> str:
169
+ return (value or "").strip().lower()
170
+
171
+
172
+ def _normalize_tag(value: str | None) -> str:
173
+ normalized = _normalize_choice(value).replace("-", "_").replace(" ", "_")
174
+ while "__" in normalized:
175
+ normalized = normalized.replace("__", "_")
176
+ return normalized
177
+
178
+
179
+ def _reply_covers_keywords(value: str | None, keywords: list[str]) -> bool:
180
+ normalized = _normalize_choice(value).replace("_", " ").replace("-", " ")
181
+ return all(_normalize_choice(keyword).replace("_", " ").replace("-", " ") in normalized for keyword in keywords)
182
+
183
+
184
  def _parse_action(content: str) -> TriageAction:
185
+ data = json.loads(_extract_json_object(content))
 
 
 
 
 
186
  return TriageAction(
187
  action_type=data["action_type"],
188
  value=data.get("value"),
 
208
  )
209
 
210
 
211
+ def _request_action_with_tools(prompt: str) -> TriageAction | None:
212
+ response = client.chat.completions.create(
213
+ model=MODEL_NAME,
214
+ messages=[{"role": "user", "content": prompt}],
215
+ temperature=0,
216
+ tools=ACTION_TOOL,
217
+ tool_choice="required",
218
+ )
219
+ action = _parse_tool_action(response)
220
+ if action is not None:
221
+ return action
222
+
223
+ content = response.choices[0].message.content or ""
224
+ return _parse_action(content)
225
+
226
+
227
+ def _request_action_with_json(prompt: str) -> TriageAction:
228
+ response = client.chat.completions.create(
229
+ model=MODEL_NAME,
230
+ messages=[
231
+ {
232
+ "role": "user",
233
+ "content": (
234
+ f"{prompt}\n"
235
+ "Return only compact JSON with keys action_type and value."
236
+ ),
237
+ }
238
+ ],
239
+ temperature=0,
240
+ )
241
+ content = response.choices[0].message.content or ""
242
+ return _parse_action(content)
243
+
244
+
245
+ def _planned_action(observation) -> TriageAction:
246
+ playbook = TASK_PLAYBOOKS.get(observation.task_id)
247
+ if playbook is None:
248
+ if observation.workspace.priority is None:
249
+ return TriageAction(action_type="set_priority", value="medium")
250
+ if observation.workspace.queue is None:
251
+ return TriageAction(action_type="set_queue", value="general_support")
252
+ if observation.workspace.draft_reply is None:
253
+ return TriageAction(
254
+ action_type="draft_reply",
255
+ value="Thanks for contacting support. We are reviewing this now.",
256
+ )
257
+ return TriageAction(action_type="submit")
258
+
259
+ workspace = observation.workspace
260
+ if _normalize_choice(workspace.priority) != playbook["priority"]:
261
+ return TriageAction(action_type="set_priority", value=playbook["priority"])
262
+ if _normalize_choice(workspace.queue) != playbook["queue"]:
263
+ return TriageAction(action_type="set_queue", value=playbook["queue"])
264
+ if _normalize_choice(workspace.status) != playbook["status"]:
265
+ return TriageAction(action_type="set_status", value=playbook["status"])
266
+
267
+ existing_tags = {_normalize_tag(tag) for tag in workspace.tags}
268
+ for tag in playbook["tags"]:
269
+ if _normalize_tag(tag) not in existing_tags:
270
+ return TriageAction(action_type="add_tag", value=tag)
271
+
272
+ if not _reply_covers_keywords(workspace.draft_reply, observation.reply_requirements):
273
+ return TriageAction(action_type="draft_reply", value=playbook["reply"])
274
+
275
  return TriageAction(action_type="submit")
276
 
277
 
278
+ def _prefer_model_action(observation, candidate: TriageAction, planned: TriageAction) -> bool:
279
+ if candidate.action_type != planned.action_type:
280
+ return False
281
+
282
+ if candidate.action_type in {"set_priority", "set_queue", "set_status"}:
283
+ return _normalize_choice(candidate.value) == _normalize_choice(planned.value)
284
+
285
+ if candidate.action_type in {"add_tag", "remove_tag"}:
286
+ return _normalize_tag(candidate.value) == _normalize_tag(planned.value)
287
+
288
+ if candidate.action_type == "draft_reply":
289
+ return _reply_covers_keywords(candidate.value, observation.reply_requirements)
290
+
291
+ return candidate.action_type == "submit"
292
+
293
+
294
  def _run_task(task_id: str) -> tuple[bool, list[float]]:
295
  env = SupportTriageEnvironment()
296
  rewards: list[float] = []
297
  success = False
298
+ score = 0.0
299
  steps = 0
300
  recent_steps: list[dict[str, str]] = []
301
 
 
305
  observation = env.reset(task_id=task_id)
306
  while not observation.done:
307
  prompt = _build_prompt_with_history(observation, recent_steps)
308
+ action = _planned_action(observation)
309
  try:
310
+ candidate = _request_action_with_tools(prompt)
 
 
 
 
 
 
 
 
 
 
 
 
311
  except Exception:
312
+ try:
313
+ candidate = _request_action_with_json(prompt)
314
+ except Exception:
315
+ candidate = None
316
+
317
+ if candidate is not None and _prefer_model_action(observation, candidate, action):
318
+ action = candidate
319
 
320
  observation = env.step(action)
321
  steps += 1
 
335
  }
336
  )
337
 
338
+ score = max(0.0, min(1.0, float(env.state.completion_score or 0.0)))
339
+ success = env.state.submitted and score >= 0.85
340
  return success, rewards
341
  finally:
342
+ score = max(0.0, min(1.0, float(env.state.completion_score or 0.0)))
343
  env.close()
344
  joined_rewards = ",".join(_format_reward(value) for value in rewards)
345
  print(
346
+ f"[END] success={str(success).lower()} steps={steps} score={score:.2f} rewards={joined_rewards}",
347
  flush=True,
348
  )
349
 
 
352
  selected_task_ids = task_ids()
353
  if TASK_IDS_OVERRIDE:
354
  selected_task_ids = [task_id.strip() for task_id in TASK_IDS_OVERRIDE.split(",") if task_id.strip()]
355
+ if not selected_task_ids:
356
+ raise ValueError("No task IDs selected. Set TASK_IDS to a comma-separated list or leave it unset.")
357
 
358
  for current_task_id in selected_task_ids:
359
  _run_task(current_task_id)
pre_validation_script.sh ADDED
@@ -0,0 +1,185 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env bash
2
+ #
3
+ # validate-submission.sh — OpenEnv Submission Validator
4
+ #
5
+ # Checks that your HF Space is live, Docker image builds, and openenv validate passes.
6
+ #
7
+ # Prerequisites:
8
+ # - Docker: https://docs.docker.com/get-docker/
9
+ # - openenv-core: pip install openenv-core
10
+ # - curl (usually pre-installed)
11
+ #
12
+ # Run:
13
+ # curl -fsSL https://raw.githubusercontent.com/<owner>/<repo>/main/scripts/validate-submission.sh | bash -s -- <ping_url> [repo_dir]
14
+ #
15
+ # Or download and run locally:
16
+ # chmod +x validate-submission.sh
17
+ # ./validate-submission.sh <ping_url> [repo_dir]
18
+ #
19
+ # Arguments:
20
+ # ping_url Your HuggingFace Space URL (e.g. https://your-space.hf.space)
21
+ # repo_dir Path to your repo (default: current directory)
22
+ #
23
+ # Examples:
24
+ # ./validate-submission.sh https://my-team.hf.space
25
+ # ./validate-submission.sh https://my-team.hf.space ./my-repo
26
+ #
27
+
28
+ set -uo pipefail
29
+
30
+ DOCKER_BUILD_TIMEOUT=600
31
+ if [ -t 1 ]; then
32
+ RED='\033[0;31m'
33
+ GREEN='\033[0;32m'
34
+ YELLOW='\033[1;33m'
35
+ BOLD='\033[1m'
36
+ NC='\033[0m'
37
+ else
38
+ RED='' GREEN='' YELLOW='' BOLD='' NC=''
39
+ fi
40
+
41
+ run_with_timeout() {
42
+ local secs="$1"; shift
43
+ if command -v timeout &>/dev/null; then
44
+ timeout "$secs" "$@"
45
+ elif command -v gtimeout &>/dev/null; then
46
+ gtimeout "$secs" "$@"
47
+ else
48
+ "$@" &
49
+ local pid=$!
50
+ ( sleep "$secs" && kill "$pid" 2>/dev/null ) &
51
+ local watcher=$!
52
+ wait "$pid" 2>/dev/null
53
+ local rc=$?
54
+ kill "$watcher" 2>/dev/null
55
+ wait "$watcher" 2>/dev/null
56
+ return $rc
57
+ fi
58
+ }
59
+
60
+ portable_mktemp() {
61
+ local prefix="${1:-validate}"
62
+ mktemp "${TMPDIR:-/tmp}/${prefix}-XXXXXX" 2>/dev/null || mktemp
63
+ }
64
+
65
+ CLEANUP_FILES=()
66
+ cleanup() { rm -f "${CLEANUP_FILES[@]+"${CLEANUP_FILES[@]}"}"; }
67
+ trap cleanup EXIT
68
+
69
+ PING_URL="${1:-}"
70
+ REPO_DIR="${2:-.}"
71
+
72
+ if [ -z "$PING_URL" ]; then
73
+ printf "Usage: %s <ping_url> [repo_dir]\n" "$0"
74
+ printf "\n"
75
+ printf " ping_url Your HuggingFace Space URL (e.g. https://your-space.hf.space)\n"
76
+ printf " repo_dir Path to your repo (default: current directory)\n"
77
+ exit 1
78
+ fi
79
+
80
+ if ! REPO_DIR="$(cd "$REPO_DIR" 2>/dev/null && pwd)"; then
81
+ printf "Error: directory '%s' not found\n" "${2:-.}"
82
+ exit 1
83
+ fi
84
+ PING_URL="${PING_URL%/}"
85
+ export PING_URL
86
+ PASS=0
87
+
88
+ log() { printf "[%s] %b\n" "$(date -u +%H:%M:%S)" "$*"; }
89
+ pass() { log "${GREEN}PASSED${NC} -- $1"; PASS=$((PASS + 1)); }
90
+ fail() { log "${RED}FAILED${NC} -- $1"; }
91
+ hint() { printf " ${YELLOW}Hint:${NC} %b\n" "$1"; }
92
+ stop_at() {
93
+ printf "\n"
94
+ printf "${RED}${BOLD}Validation stopped at %s.${NC} Fix the above before continuing.\n" "$1"
95
+ exit 1
96
+ }
97
+
98
+ printf "\n"
99
+ printf "${BOLD}========================================${NC}\n"
100
+ printf "${BOLD} OpenEnv Submission Validator${NC}\n"
101
+ printf "${BOLD}========================================${NC}\n"
102
+ log "Repo: $REPO_DIR"
103
+ log "Ping URL: $PING_URL"
104
+ printf "\n"
105
+
106
+ log "${BOLD}Step 1/3: Pinging HF Space${NC} ($PING_URL/reset) ..."
107
+
108
+ CURL_OUTPUT=$(portable_mktemp "validate-curl")
109
+ CLEANUP_FILES+=("$CURL_OUTPUT")
110
+ HTTP_CODE=$(curl -s -o "$CURL_OUTPUT" -w "%{http_code}" -X POST \
111
+ -H "Content-Type: application/json" -d '{}' \
112
+ "$PING_URL/reset" --max-time 30 2>"$CURL_OUTPUT" || printf "000")
113
+
114
+ if [ "$HTTP_CODE" = "200" ]; then
115
+ pass "HF Space is live and responds to /reset"
116
+ elif [ "$HTTP_CODE" = "000" ]; then
117
+ fail "HF Space not reachable (connection failed or timed out)"
118
+ hint "Check your network connection and that the Space is running."
119
+ hint "Try: curl -s -o /dev/null -w '%%{http_code}' -X POST $PING_URL/reset"
120
+ stop_at "Step 1"
121
+ else
122
+ fail "HF Space /reset returned HTTP $HTTP_CODE (expected 200)"
123
+ hint "Make sure your Space is running and the URL is correct."
124
+ hint "Try opening $PING_URL in your browser first."
125
+ stop_at "Step 1"
126
+ fi
127
+
128
+ log "${BOLD}Step 2/3: Running docker build${NC} ..."
129
+
130
+ if ! command -v docker &>/dev/null; then
131
+ fail "docker command not found"
132
+ hint "Install Docker: https://docs.docker.com/get-docker/"
133
+ stop_at "Step 2"
134
+ fi
135
+
136
+ if [ -f "$REPO_DIR/Dockerfile" ]; then
137
+ DOCKER_CONTEXT="$REPO_DIR"
138
+ elif [ -f "$REPO_DIR/server/Dockerfile" ]; then
139
+ DOCKER_CONTEXT="$REPO_DIR/server"
140
+ else
141
+ fail "No Dockerfile found in repo root or server/ directory"
142
+ stop_at "Step 2"
143
+ fi
144
+
145
+ log " Found Dockerfile in $DOCKER_CONTEXT"
146
+
147
+ BUILD_OK=false
148
+ BUILD_OUTPUT=$(run_with_timeout "$DOCKER_BUILD_TIMEOUT" docker build "$DOCKER_CONTEXT" 2>&1) && BUILD_OK=true
149
+
150
+ if [ "$BUILD_OK" = true ]; then
151
+ pass "Docker build succeeded"
152
+ else
153
+ fail "Docker build failed (timeout=${DOCKER_BUILD_TIMEOUT}s)"
154
+ printf "%s\n" "$BUILD_OUTPUT" | tail -20
155
+ stop_at "Step 2"
156
+ fi
157
+
158
+ log "${BOLD}Step 3/3: Running openenv validate${NC} ..."
159
+
160
+ if ! command -v openenv &>/dev/null; then
161
+ fail "openenv command not found"
162
+ hint "Install it: pip install openenv-core"
163
+ stop_at "Step 3"
164
+ fi
165
+
166
+ VALIDATE_OK=false
167
+ VALIDATE_OUTPUT=$(cd "$REPO_DIR" && openenv validate 2>&1) && VALIDATE_OK=true
168
+
169
+ if [ "$VALIDATE_OK" = true ]; then
170
+ pass "openenv validate passed"
171
+ [ -n "$VALIDATE_OUTPUT" ] && log " $VALIDATE_OUTPUT"
172
+ else
173
+ fail "openenv validate failed"
174
+ printf "%s\n" "$VALIDATE_OUTPUT"
175
+ stop_at "Step 3"
176
+ fi
177
+
178
+ printf "\n"
179
+ printf "${BOLD}========================================${NC}\n"
180
+ printf "${GREEN}${BOLD} All 3/3 checks passed!${NC}\n"
181
+ printf "${GREEN}${BOLD} Your submission is ready to submit.${NC}\n"
182
+ printf "${BOLD}========================================${NC}\n"
183
+ printf "\n"
184
+
185
+ exit 0
sample_inference.py ADDED
@@ -0,0 +1,188 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Inference Script Example
3
+ ===================================
4
+ MANDATORY
5
+ - Before submitting, ensure the following variables are defined in your environment configuration:
6
+ API_BASE_URL The API endpoint for the LLM.
7
+ MODEL_NAME The model identifier to use for inference.
8
+ HF_TOKEN Your Hugging Face / API key.
9
+ LOCAL_IMAGE_NAME The name of the local image to use for the environment if you are using from_docker_image()
10
+ method
11
+
12
+ - Defaults are set only for API_BASE_URL and MODEL_NAME
13
+ (and should reflect your active inference setup):
14
+ API_BASE_URL = os.getenv("API_BASE_URL", "<your-active-endpoint>")
15
+ MODEL_NAME = os.getenv("MODEL_NAME", "<your-active-model>")
16
+
17
+ - The inference script must be named `inference.py` and placed in the root directory of the project
18
+ - Participants must use OpenAI Client for all LLM calls using above variables
19
+
20
+ STDOUT FORMAT
21
+ - The script must emit exactly three line types to stdout, in this order:
22
+
23
+ [START] task=<task_name> env=<benchmark> model=<model_name>
24
+ [STEP] step=<n> action=<action_str> reward=<0.00> done=<true|false> error=<msg|null>
25
+ [END] success=<true|false> steps=<n> score=<score> rewards=<r1,r2,...,rn>
26
+
27
+ Rules:
28
+ - One [START] line at episode begin.
29
+ - One [STEP] line per step, immediately after env.step() returns.
30
+ - One [END] line after env.close(), always emitted (even on exception).
31
+ - reward and rewards are formatted to 2 decimal places.
32
+ - done and success are lowercase booleans: true or false.
33
+ - error is the raw last_action_error string, or null if none.
34
+ - All fields on a single line with no newlines within a line.
35
+ - Each tasks should return score in [0, 1]
36
+
37
+ Example:
38
+ [START] task=click-test env=miniwob model=Qwen3-VL-30B
39
+ [STEP] step=1 action=click('123') reward=0.00 done=false error=null
40
+ [STEP] step=2 action=fill('456','text') reward=0.00 done=false error=null
41
+ [STEP] step=3 action=click('789') reward=1.00 done=true error=null
42
+ [END] success=true steps=3 score=1.00 rewards=0.00,0.00,1.00
43
+ """
44
+
45
+ import asyncio
46
+ import os
47
+ import textwrap
48
+ from typing import List, Optional
49
+
50
+ from openai import OpenAI
51
+
52
+ from my_env_v4 import MyEnvV4Action, MyEnvV4Env
53
+ IMAGE_NAME = os.getenv("IMAGE_NAME") # If you are using docker image
54
+ API_KEY = os.getenv("HF_TOKEN") or os.getenv("API_KEY")
55
+
56
+ API_BASE_URL = os.getenv("API_BASE_URL") or "https://router.huggingface.co/v1"
57
+ MODEL_NAME = os.getenv("MODEL_NAME") or "Qwen/Qwen2.5-72B-Instruct"
58
+ TASK_NAME = os.getenv("MY_ENV_V4_TASK", "echo")
59
+ BENCHMARK = os.getenv("MY_ENV_V4_BENCHMARK", "my_env_v4")
60
+ MAX_STEPS = 8
61
+ TEMPERATURE = 0.7
62
+ MAX_TOKENS = 150
63
+ SUCCESS_SCORE_THRESHOLD = 0.1 # normalized score in [0, 1]
64
+
65
+ # Max possible reward: each token contributes 0.1, across all steps
66
+ _MAX_REWARD_PER_STEP = MAX_TOKENS * 0.1
67
+ MAX_TOTAL_REWARD = MAX_STEPS * _MAX_REWARD_PER_STEP
68
+
69
+ SYSTEM_PROMPT = textwrap.dedent(
70
+ """
71
+ You are interacting with a simple echo environment.
72
+ Each turn you must send a message. The environment will echo it back.
73
+ Reward is proportional to message length: reward = len(message) * 0.1
74
+ Your goal is to maximize total reward by sending meaningful, substantive messages.
75
+ Reply with exactly one message string — no quotes, no prefixes, just the message text.
76
+ """
77
+ ).strip()
78
+
79
+
80
+ def log_start(task: str, env: str, model: str) -> None:
81
+ print(f"[START] task={task} env={env} model={model}", flush=True)
82
+
83
+
84
+ def log_step(step: int, action: str, reward: float, done: bool, error: Optional[str]) -> None:
85
+ error_val = error if error else "null"
86
+ done_val = str(done).lower()
87
+ print(
88
+ f"[STEP] step={step} action={action} reward={reward:.2f} done={done_val} error={error_val}",
89
+ flush=True,
90
+ )
91
+
92
+
93
+ def log_end(success: bool, steps: int, score: float, rewards: List[float]) -> None:
94
+ rewards_str = ",".join(f"{r:.2f}" for r in rewards)
95
+ print(f"[END] success={str(success).lower()} steps={steps} score={score:.3f} rewards={rewards_str}", flush=True)
96
+
97
+
98
+ def build_user_prompt(step: int, last_echoed: str, last_reward: float, history: List[str]) -> str:
99
+ history_block = "\n".join(history[-4:]) if history else "None"
100
+ return textwrap.dedent(
101
+ f"""
102
+ Step: {step}
103
+ Last echoed message: {last_echoed!r}
104
+ Last reward: {last_reward:.2f}
105
+ Previous steps:
106
+ {history_block}
107
+ Send your next message.
108
+ """
109
+ ).strip()
110
+
111
+
112
+ def get_model_message(client: OpenAI, step: int, last_echoed: str, last_reward: float, history: List[str]) -> str:
113
+ user_prompt = build_user_prompt(step, last_echoed, last_reward, history)
114
+ try:
115
+ completion = client.chat.completions.create(
116
+ model=MODEL_NAME,
117
+ messages=[
118
+ {"role": "system", "content": SYSTEM_PROMPT},
119
+ {"role": "user", "content": user_prompt},
120
+ ],
121
+ temperature=TEMPERATURE,
122
+ max_tokens=MAX_TOKENS,
123
+ stream=False,
124
+ )
125
+ text = (completion.choices[0].message.content or "").strip()
126
+ return text if text else "hello"
127
+ except Exception as exc:
128
+ print(f"[DEBUG] Model request failed: {exc}", flush=True)
129
+ return "hello"
130
+
131
+
132
+ async def main() -> None:
133
+ client = OpenAI(base_url=API_BASE_URL, api_key=API_KEY)
134
+
135
+ env = await MyEnvV4Env.from_docker_image(IMAGE_NAME)
136
+
137
+ history: List[str] = []
138
+ rewards: List[float] = []
139
+ steps_taken = 0
140
+ score = 0.0
141
+ success = False
142
+
143
+ log_start(task=TASK_NAME, env=BENCHMARK, model=MODEL_NAME)
144
+
145
+ try:
146
+ result = await env.reset() # OpenENV.reset()
147
+ last_echoed = result.observation.echoed_message
148
+ last_reward = 0.0
149
+
150
+ for step in range(1, MAX_STEPS + 1):
151
+ if result.done:
152
+ break
153
+
154
+ message = get_model_message(client, step, last_echoed, last_reward, history)
155
+
156
+ result = await env.step(MyEnvV4Action(message=message))
157
+ obs = result.observation
158
+
159
+ reward = result.reward or 0.0
160
+ done = result.done
161
+ error = None
162
+
163
+ rewards.append(reward)
164
+ steps_taken = step
165
+ last_echoed = obs.echoed_message
166
+ last_reward = reward
167
+
168
+ log_step(step=step, action=message, reward=reward, done=done, error=error)
169
+
170
+ history.append(f"Step {step}: {message!r} -> reward {reward:+.2f}")
171
+
172
+ if done:
173
+ break
174
+
175
+ score = sum(rewards) / MAX_TOTAL_REWARD if MAX_TOTAL_REWARD > 0 else 0.0
176
+ score = min(max(score, 0.0), 1.0) # clamp to [0, 1]
177
+ success = score >= SUCCESS_SCORE_THRESHOLD
178
+
179
+ finally:
180
+ try:
181
+ await env.close()
182
+ except Exception as e:
183
+ print(f"[DEBUG] env.close() error (container cleanup): {e}", flush=True)
184
+ log_end(success=success, steps=steps_taken, score=score, rewards=rewards)
185
+
186
+
187
+ if __name__ == "__main__":
188
+ asyncio.run(main())
tests/test_inference.py ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import importlib
2
+ import sys
3
+
4
+
5
+ def _load_inference(monkeypatch):
6
+ monkeypatch.setenv("HF_TOKEN", "test-token")
7
+ monkeypatch.setenv("API_BASE_URL", "https://api.openai.com/v1")
8
+ monkeypatch.setenv("MODEL_NAME", "test-model")
9
+ sys.modules.pop("inference", None)
10
+ return importlib.import_module("inference")
11
+
12
+
13
+ def test_parse_action_accepts_wrapped_json(monkeypatch):
14
+ inference = _load_inference(monkeypatch)
15
+
16
+ action = inference._parse_action(
17
+ "Use this action:\n```json\n{\"action_type\":\"submit\",\"value\":null}\n```"
18
+ )
19
+
20
+ assert action.action_type == "submit"
21
+ assert action.value is None
22
+
23
+
24
+ def test_rule_based_fallback_solves_all_tasks(monkeypatch, capsys):
25
+ inference = _load_inference(monkeypatch)
26
+
27
+ def _raise_provider_error(prompt: str):
28
+ del prompt
29
+ raise RuntimeError("provider unavailable")
30
+
31
+ monkeypatch.setattr(inference, "_request_action_with_tools", _raise_provider_error)
32
+ monkeypatch.setattr(inference, "_request_action_with_json", _raise_provider_error)
33
+
34
+ for task_id in ("billing_duplicate_charge", "shipping_damaged_replacement", "account_takeover_fraud"):
35
+ success, rewards = inference._run_task(task_id)
36
+ output = capsys.readouterr().out
37
+
38
+ assert success is True
39
+ assert rewards
40
+ assert f"[START] task={task_id} env=support_triage_env model=test-model" in output
41
+ assert "[END] success=true" in output
42
+ assert "score=1.00" in output