exploring-solver commited on
Commit
57db7b7
·
1 Parent(s): e849359

Validator 0-1 score change

Browse files
Pre_Validation_Script.sh DELETED
@@ -1,185 +0,0 @@
1
- #!/usr/bin/env bash
2
- #
3
- # validate-submission.sh — OpenEnv Submission Validator
4
- #
5
- # Checks that your HF Space is live, Docker image builds, and openenv validate passes.
6
- #
7
- # Prerequisites:
8
- # - Docker: https://docs.docker.com/get-docker/
9
- # - openenv-core: pip install openenv-core
10
- # - curl (usually pre-installed)
11
- #
12
- # Run:
13
- # curl -fsSL https://raw.githubusercontent.com/<owner>/<repo>/main/scripts/validate-submission.sh | bash -s -- <ping_url> [repo_dir]
14
- #
15
- # Or download and run locally:
16
- # chmod +x validate-submission.sh
17
- # ./validate-submission.sh <ping_url> [repo_dir]
18
- #
19
- # Arguments:
20
- # ping_url Your HuggingFace Space URL (e.g. https://your-space.hf.space)
21
- # repo_dir Path to your repo (default: current directory)
22
- #
23
- # Examples:
24
- # ./validate-submission.sh https://my-team.hf.space
25
- # ./validate-submission.sh https://my-team.hf.space ./my-repo
26
- #
27
-
28
- set -uo pipefail
29
-
30
- DOCKER_BUILD_TIMEOUT=600
31
- if [ -t 1 ]; then
32
- RED='\033[0;31m'
33
- GREEN='\033[0;32m'
34
- YELLOW='\033[1;33m'
35
- BOLD='\033[1m'
36
- NC='\033[0m'
37
- else
38
- RED='' GREEN='' YELLOW='' BOLD='' NC=''
39
- fi
40
-
41
- run_with_timeout() {
42
- local secs="$1"; shift
43
- if command -v timeout &>/dev/null; then
44
- timeout "$secs" "$@"
45
- elif command -v gtimeout &>/dev/null; then
46
- gtimeout "$secs" "$@"
47
- else
48
- "$@" &
49
- local pid=$!
50
- ( sleep "$secs" && kill "$pid" 2>/dev/null ) &
51
- local watcher=$!
52
- wait "$pid" 2>/dev/null
53
- local rc=$?
54
- kill "$watcher" 2>/dev/null
55
- wait "$watcher" 2>/dev/null
56
- return $rc
57
- fi
58
- }
59
-
60
- portable_mktemp() {
61
- local prefix="${1:-validate}"
62
- mktemp "${TMPDIR:-/tmp}/${prefix}-XXXXXX" 2>/dev/null || mktemp
63
- }
64
-
65
- CLEANUP_FILES=()
66
- cleanup() { rm -f "${CLEANUP_FILES[@]+"${CLEANUP_FILES[@]}"}"; }
67
- trap cleanup EXIT
68
-
69
- PING_URL="${1:-}"
70
- REPO_DIR="${2:-.}"
71
-
72
- if [ -z "$PING_URL" ]; then
73
- printf "Usage: %s <ping_url> [repo_dir]\n" "$0"
74
- printf "\n"
75
- printf " ping_url Your HuggingFace Space URL (e.g. https://your-space.hf.space)\n"
76
- printf " repo_dir Path to your repo (default: current directory)\n"
77
- exit 1
78
- fi
79
-
80
- if ! REPO_DIR="$(cd "$REPO_DIR" 2>/dev/null && pwd)"; then
81
- printf "Error: directory '%s' not found\n" "${2:-.}"
82
- exit 1
83
- fi
84
- PING_URL="${PING_URL%/}"
85
- export PING_URL
86
- PASS=0
87
-
88
- log() { printf "[%s] %b\n" "$(date -u +%H:%M:%S)" "$*"; }
89
- pass() { log "${GREEN}PASSED${NC} -- $1"; PASS=$((PASS + 1)); }
90
- fail() { log "${RED}FAILED${NC} -- $1"; }
91
- hint() { printf " ${YELLOW}Hint:${NC} %b\n" "$1"; }
92
- stop_at() {
93
- printf "\n"
94
- printf "${RED}${BOLD}Validation stopped at %s.${NC} Fix the above before continuing.\n" "$1"
95
- exit 1
96
- }
97
-
98
- printf "\n"
99
- printf "${BOLD}========================================${NC}\n"
100
- printf "${BOLD} OpenEnv Submission Validator${NC}\n"
101
- printf "${BOLD}========================================${NC}\n"
102
- log "Repo: $REPO_DIR"
103
- log "Ping URL: $PING_URL"
104
- printf "\n"
105
-
106
- log "${BOLD}Step 1/3: Pinging HF Space${NC} ($PING_URL/reset) ..."
107
-
108
- CURL_OUTPUT=$(portable_mktemp "validate-curl")
109
- CLEANUP_FILES+=("$CURL_OUTPUT")
110
- HTTP_CODE=$(curl -s -o "$CURL_OUTPUT" -w "%{http_code}" -X POST \
111
- -H "Content-Type: application/json" -d '{}' \
112
- "$PING_URL/reset" --max-time 30 2>"$CURL_OUTPUT" || printf "000")
113
-
114
- if [ "$HTTP_CODE" = "200" ]; then
115
- pass "HF Space is live and responds to /reset"
116
- elif [ "$HTTP_CODE" = "000" ]; then
117
- fail "HF Space not reachable (connection failed or timed out)"
118
- hint "Check your network connection and that the Space is running."
119
- hint "Try: curl -s -o /dev/null -w '%%{http_code}' -X POST $PING_URL/reset"
120
- stop_at "Step 1"
121
- else
122
- fail "HF Space /reset returned HTTP $HTTP_CODE (expected 200)"
123
- hint "Make sure your Space is running and the URL is correct."
124
- hint "Try opening $PING_URL in your browser first."
125
- stop_at "Step 1"
126
- fi
127
-
128
- log "${BOLD}Step 2/3: Running docker build${NC} ..."
129
-
130
- if ! command -v docker &>/dev/null; then
131
- fail "docker command not found"
132
- hint "Install Docker: https://docs.docker.com/get-docker/"
133
- stop_at "Step 2"
134
- fi
135
-
136
- if [ -f "$REPO_DIR/Dockerfile" ]; then
137
- DOCKER_CONTEXT="$REPO_DIR"
138
- elif [ -f "$REPO_DIR/server/Dockerfile" ]; then
139
- DOCKER_CONTEXT="$REPO_DIR/server"
140
- else
141
- fail "No Dockerfile found in repo root or server/ directory"
142
- stop_at "Step 2"
143
- fi
144
-
145
- log " Found Dockerfile in $DOCKER_CONTEXT"
146
-
147
- BUILD_OK=false
148
- BUILD_OUTPUT=$(run_with_timeout "$DOCKER_BUILD_TIMEOUT" docker build "$DOCKER_CONTEXT" 2>&1) && BUILD_OK=true
149
-
150
- if [ "$BUILD_OK" = true ]; then
151
- pass "Docker build succeeded"
152
- else
153
- fail "Docker build failed (timeout=${DOCKER_BUILD_TIMEOUT}s)"
154
- printf "%s\n" "$BUILD_OUTPUT" | tail -20
155
- stop_at "Step 2"
156
- fi
157
-
158
- log "${BOLD}Step 3/3: Running openenv validate${NC} ..."
159
-
160
- if ! command -v openenv &>/dev/null; then
161
- fail "openenv command not found"
162
- hint "Install it: pip install openenv-core"
163
- stop_at "Step 3"
164
- fi
165
-
166
- VALIDATE_OK=false
167
- VALIDATE_OUTPUT=$(cd "$REPO_DIR" && openenv validate 2>&1) && VALIDATE_OK=true
168
-
169
- if [ "$VALIDATE_OK" = true ]; then
170
- pass "openenv validate passed"
171
- [ -n "$VALIDATE_OUTPUT" ] && log " $VALIDATE_OUTPUT"
172
- else
173
- fail "openenv validate failed"
174
- printf "%s\n" "$VALIDATE_OUTPUT"
175
- stop_at "Step 3"
176
- fi
177
-
178
- printf "\n"
179
- printf "${BOLD}========================================${NC}\n"
180
- printf "${GREEN}${BOLD} All 3/3 checks passed!${NC}\n"
181
- printf "${GREEN}${BOLD} Your submission is ready to submit.${NC}\n"
182
- printf "${BOLD}========================================${NC}\n"
183
- printf "\n"
184
-
185
- exit 0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
Sample_Inference_Script.py DELETED
@@ -1,187 +0,0 @@
1
- """
2
- Inference Script Example
3
- ===================================
4
- MANDATORY
5
- - Before submitting, ensure the following variables are defined in your environment configuration:
6
- API_BASE_URL The API endpoint for the LLM.
7
- MODEL_NAME The model identifier to use for inference.
8
- HF_TOKEN Your Hugging Face / API key.
9
- LOCAL_IMAGE_NAME The name of the local image to use for the environment if you are using from_docker_image()
10
- method
11
-
12
- - Defaults are set only for API_BASE_URL and MODEL_NAME
13
- (and should reflect your active inference setup):
14
- API_BASE_URL = os.getenv("API_BASE_URL", "<your-active-endpoint>")
15
- MODEL_NAME = os.getenv("MODEL_NAME", "<your-active-model>")
16
-
17
- - The inference script must be named `inference.py` and placed in the root directory of the project
18
- - Participants must use OpenAI Client for all LLM calls using above variables
19
-
20
- STDOUT FORMAT
21
- - The script must emit exactly three line types to stdout, in this order:
22
-
23
- [START] task=<task_name> env=<benchmark> model=<model_name>
24
- [STEP] step=<n> action=<action_str> reward=<0.00> done=<true|false> error=<msg|null>
25
- [END] success=<true|false> steps=<n> rewards=<r1,r2,...,rn>
26
-
27
- Rules:
28
- - One [START] line at episode begin.
29
- - One [STEP] line per step, immediately after env.step() returns.
30
- - One [END] line after env.close(), always emitted (even on exception).
31
- - reward and rewards are formatted to 2 decimal places.
32
- - done and success are lowercase booleans: true or false.
33
- - error is the raw last_action_error string, or null if none.
34
- - All fields on a single line with no newlines within a line.
35
-
36
- Example:
37
- [START] task=click-test env=miniwob model=Qwen3-VL-30B
38
- [STEP] step=1 action=click('123') reward=0.00 done=false error=null
39
- [STEP] step=2 action=fill('456','text') reward=0.00 done=false error=null
40
- [STEP] step=3 action=click('789') reward=1.00 done=true error=null
41
- [END] success=true steps=3 rewards=0.00,0.00,1.00
42
- """
43
-
44
- import asyncio
45
- import os
46
- import textwrap
47
- from typing import List, Optional
48
-
49
- from openai import OpenAI
50
-
51
- from my_env_v4 import MyEnvV4Action, MyEnvV4Env
52
- IMAGE_NAME = os.getenv("IMAGE_NAME") # If you are using docker image
53
- API_KEY = os.getenv("HF_TOKEN") or os.getenv("API_KEY")
54
-
55
- API_BASE_URL = os.getenv("API_BASE_URL") or "https://router.huggingface.co/v1"
56
- MODEL_NAME = os.getenv("MODEL_NAME") or "Qwen/Qwen2.5-72B-Instruct"
57
- TASK_NAME = os.getenv("MY_ENV_V4_TASK", "echo")
58
- BENCHMARK = os.getenv("MY_ENV_V4_BENCHMARK", "my_env_v4")
59
- MAX_STEPS = 8
60
- TEMPERATURE = 0.7
61
- MAX_TOKENS = 150
62
- SUCCESS_SCORE_THRESHOLD = 0.1 # normalized score in [0, 1]
63
-
64
- # Max possible reward: each token contributes 0.1, across all steps
65
- _MAX_REWARD_PER_STEP = MAX_TOKENS * 0.1
66
- MAX_TOTAL_REWARD = MAX_STEPS * _MAX_REWARD_PER_STEP
67
-
68
- SYSTEM_PROMPT = textwrap.dedent(
69
- """
70
- You are interacting with a simple echo environment.
71
- Each turn you must send a message. The environment will echo it back.
72
- Reward is proportional to message length: reward = len(message) * 0.1
73
- Your goal is to maximize total reward by sending meaningful, substantive messages.
74
- Reply with exactly one message string — no quotes, no prefixes, just the message text.
75
- """
76
- ).strip()
77
-
78
-
79
- def log_start(task: str, env: str, model: str) -> None:
80
- print(f"[START] task={task} env={env} model={model}", flush=True)
81
-
82
-
83
- def log_step(step: int, action: str, reward: float, done: bool, error: Optional[str]) -> None:
84
- error_val = error if error else "null"
85
- done_val = str(done).lower()
86
- print(
87
- f"[STEP] step={step} action={action} reward={reward:.2f} done={done_val} error={error_val}",
88
- flush=True,
89
- )
90
-
91
-
92
- def log_end(success: bool, steps: int, score: float, rewards: List[float]) -> None:
93
- rewards_str = ",".join(f"{r:.2f}" for r in rewards)
94
- print(f"[END] success={str(success).lower()} steps={steps} score={score:.3f} rewards={rewards_str}", flush=True)
95
-
96
-
97
- def build_user_prompt(step: int, last_echoed: str, last_reward: float, history: List[str]) -> str:
98
- history_block = "\n".join(history[-4:]) if history else "None"
99
- return textwrap.dedent(
100
- f"""
101
- Step: {step}
102
- Last echoed message: {last_echoed!r}
103
- Last reward: {last_reward:.2f}
104
- Previous steps:
105
- {history_block}
106
- Send your next message.
107
- """
108
- ).strip()
109
-
110
-
111
- def get_model_message(client: OpenAI, step: int, last_echoed: str, last_reward: float, history: List[str]) -> str:
112
- user_prompt = build_user_prompt(step, last_echoed, last_reward, history)
113
- try:
114
- completion = client.chat.completions.create(
115
- model=MODEL_NAME,
116
- messages=[
117
- {"role": "system", "content": SYSTEM_PROMPT},
118
- {"role": "user", "content": user_prompt},
119
- ],
120
- temperature=TEMPERATURE,
121
- max_tokens=MAX_TOKENS,
122
- stream=False,
123
- )
124
- text = (completion.choices[0].message.content or "").strip()
125
- return text if text else "hello"
126
- except Exception as exc:
127
- print(f"[DEBUG] Model request failed: {exc}", flush=True)
128
- return "hello"
129
-
130
-
131
- async def main() -> None:
132
- client = OpenAI(base_url=API_BASE_URL, api_key=API_KEY)
133
-
134
- env = await MyEnvV4Env.from_docker_image(IMAGE_NAME)
135
-
136
- history: List[str] = []
137
- rewards: List[float] = []
138
- steps_taken = 0
139
- score = 0.0
140
- success = False
141
-
142
- log_start(task=TASK_NAME, env=BENCHMARK, model=MODEL_NAME)
143
-
144
- try:
145
- result = await env.reset() # OpenENV.reset()
146
- last_echoed = result.observation.echoed_message
147
- last_reward = 0.0
148
-
149
- for step in range(1, MAX_STEPS + 1):
150
- if result.done:
151
- break
152
-
153
- message = get_model_message(client, step, last_echoed, last_reward, history)
154
-
155
- result = await env.step(MyEnvV4Action(message=message))
156
- obs = result.observation
157
-
158
- reward = result.reward or 0.0
159
- done = result.done
160
- error = None
161
-
162
- rewards.append(reward)
163
- steps_taken = step
164
- last_echoed = obs.echoed_message
165
- last_reward = reward
166
-
167
- log_step(step=step, action=message, reward=reward, done=done, error=error)
168
-
169
- history.append(f"Step {step}: {message!r} -> reward {reward:+.2f}")
170
-
171
- if done:
172
- break
173
-
174
- score = sum(rewards) / MAX_TOTAL_REWARD if MAX_TOTAL_REWARD > 0 else 0.0
175
- score = min(max(score, 0.0), 1.0) # clamp to [0, 1]
176
- success = score >= SUCCESS_SCORE_THRESHOLD
177
-
178
- finally:
179
- try:
180
- await env.close()
181
- except Exception as e:
182
- print(f"[DEBUG] env.close() error (container cleanup): {e}", flush=True)
183
- log_end(success=success, steps=steps_taken, score=score, rewards=rewards)
184
-
185
-
186
- if __name__ == "__main__":
187
- asyncio.run(main())
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
graders.py CHANGED
@@ -2,7 +2,7 @@
2
  Deterministic graders for SupportEnv tasks.
3
 
4
  Each grader inspects the agent's action_history against ground-truth data
5
- and returns (score, breakdown, feedback) where score is in [0.0, 1.0].
6
 
7
  Task 1 — Classification: category match (0.50) + priority match (0.40) + efficiency (0.10)
8
  Task 2 — Extraction: entity coverage (0.60) + action coverage (0.30) + no hallucination (0.10)
@@ -14,6 +14,14 @@ from __future__ import annotations
14
  from typing import Any, Dict, List, Optional, Tuple
15
 
16
 
 
 
 
 
 
 
 
 
17
  def grade_task(
18
  task_id: str, episode_state: Dict[str, Any]
19
  ) -> Tuple[float, Dict[str, float], str]:
@@ -23,7 +31,7 @@ def grade_task(
23
  return _grade_extraction(episode_state)
24
  elif task_id == "task3":
25
  return _grade_resolution(episode_state)
26
- return 0.0, {}, "Unknown task"
27
 
28
 
29
  # ---------------------------------------------------------------------------
@@ -66,7 +74,7 @@ def _grade_classification(ep: Dict[str, Any]) -> Tuple[float, Dict[str, float],
66
 
67
  classify_action = _last_action_of_type(history, "classify")
68
  if classify_action is None:
69
- return 0.0, breakdown, "No classify action found."
70
 
71
  # Category
72
  if _normalize(classify_action.get("category")) == _normalize(gt["category"]):
@@ -84,7 +92,7 @@ def _grade_classification(ep: Dict[str, Any]) -> Tuple[float, Dict[str, float],
84
  else:
85
  breakdown["efficiency"] = round(max(0.0, 0.10 * (1 - (steps_used - 1) / max_steps)), 4)
86
 
87
- score = round(min(sum(breakdown.values()), 1.0), 4)
88
  parts = ", ".join(f"{k}={v:.2f}" for k, v in breakdown.items())
89
  return score, breakdown, f"Task 1: {parts}"
90
 
@@ -112,7 +120,7 @@ def _grade_extraction(ep: Dict[str, Any]) -> Tuple[float, Dict[str, float], str]
112
  extract_action = _last_action_of_type(history, "extract")
113
  if extract_action is None:
114
  breakdown["no_hallucination"] = 0.0
115
- return 0.0, breakdown, "No extract action found."
116
 
117
  # --- Entity coverage ---
118
  gt_entities: Dict[str, Any] = gt.get("entities", {})
@@ -144,7 +152,7 @@ def _grade_extraction(ep: Dict[str, Any]) -> Tuple[float, Dict[str, float], str]
144
  penalty = min(len(extra_keys) * 0.02, 0.10)
145
  breakdown["no_hallucination"] = round(max(0.0, 0.10 - penalty), 4)
146
 
147
- score = round(min(sum(breakdown.values()), 1.0), 4)
148
  parts = ", ".join(f"{k}={v:.2f}" for k, v in breakdown.items())
149
  return score, breakdown, f"Task 2: {parts}"
150
 
@@ -185,7 +193,7 @@ def _grade_resolution(ep: Dict[str, Any]) -> Tuple[float, Dict[str, float], str]
185
  respond_action = _last_action_of_type(history, "respond")
186
  if respond_action is None:
187
  breakdown["no_empty_steps"] = 0.0
188
- return 0.0, breakdown, "No respond action found."
189
 
190
  response_text: str = respond_action.get("response_text") or ""
191
  resolution_steps: List[str] = respond_action.get("resolution_steps") or []
@@ -239,6 +247,6 @@ def _grade_resolution(ep: Dict[str, Any]) -> Tuple[float, Dict[str, float], str]
239
  if not resolution_steps or any(not s.strip() for s in resolution_steps):
240
  breakdown["no_empty_steps"] = 0.0
241
 
242
- score = round(min(sum(breakdown.values()), 1.0), 4)
243
  parts = ", ".join(f"{k}={v:.2f}" for k, v in breakdown.items())
244
  return score, breakdown, f"Task 3: {parts}"
 
2
  Deterministic graders for SupportEnv tasks.
3
 
4
  Each grader inspects the agent's action_history against ground-truth data
5
+ and returns (score, breakdown, feedback) where score is in (0.0, 1.0).
6
 
7
  Task 1 — Classification: category match (0.50) + priority match (0.40) + efficiency (0.10)
8
  Task 2 — Extraction: entity coverage (0.60) + action coverage (0.30) + no hallucination (0.10)
 
14
  from typing import Any, Dict, List, Optional, Tuple
15
 
16
 
17
+ SCORE_EPSILON = 0.0001
18
+
19
+
20
+ def _strict_score(score: float) -> float:
21
+ """Map any score into the strict open interval (0, 1)."""
22
+ return round(min(max(score, SCORE_EPSILON), 1.0 - SCORE_EPSILON), 4)
23
+
24
+
25
  def grade_task(
26
  task_id: str, episode_state: Dict[str, Any]
27
  ) -> Tuple[float, Dict[str, float], str]:
 
31
  return _grade_extraction(episode_state)
32
  elif task_id == "task3":
33
  return _grade_resolution(episode_state)
34
+ return _strict_score(0.0), {}, "Unknown task"
35
 
36
 
37
  # ---------------------------------------------------------------------------
 
74
 
75
  classify_action = _last_action_of_type(history, "classify")
76
  if classify_action is None:
77
+ return _strict_score(0.0), breakdown, "No classify action found."
78
 
79
  # Category
80
  if _normalize(classify_action.get("category")) == _normalize(gt["category"]):
 
92
  else:
93
  breakdown["efficiency"] = round(max(0.0, 0.10 * (1 - (steps_used - 1) / max_steps)), 4)
94
 
95
+ score = _strict_score(sum(breakdown.values()))
96
  parts = ", ".join(f"{k}={v:.2f}" for k, v in breakdown.items())
97
  return score, breakdown, f"Task 1: {parts}"
98
 
 
120
  extract_action = _last_action_of_type(history, "extract")
121
  if extract_action is None:
122
  breakdown["no_hallucination"] = 0.0
123
+ return _strict_score(0.0), breakdown, "No extract action found."
124
 
125
  # --- Entity coverage ---
126
  gt_entities: Dict[str, Any] = gt.get("entities", {})
 
152
  penalty = min(len(extra_keys) * 0.02, 0.10)
153
  breakdown["no_hallucination"] = round(max(0.0, 0.10 - penalty), 4)
154
 
155
+ score = _strict_score(sum(breakdown.values()))
156
  parts = ", ".join(f"{k}={v:.2f}" for k, v in breakdown.items())
157
  return score, breakdown, f"Task 2: {parts}"
158
 
 
193
  respond_action = _last_action_of_type(history, "respond")
194
  if respond_action is None:
195
  breakdown["no_empty_steps"] = 0.0
196
+ return _strict_score(0.0), breakdown, "No respond action found."
197
 
198
  response_text: str = respond_action.get("response_text") or ""
199
  resolution_steps: List[str] = respond_action.get("resolution_steps") or []
 
247
  if not resolution_steps or any(not s.strip() for s in resolution_steps):
248
  breakdown["no_empty_steps"] = 0.0
249
 
250
+ score = _strict_score(sum(breakdown.values()))
251
  parts = ", ".join(f"{k}={v:.2f}" for k, v in breakdown.items())
252
  return score, breakdown, f"Task 3: {parts}"
test_integration.py DELETED
@@ -1,112 +0,0 @@
1
- """
2
- Integration test for SupportEnv.
3
-
4
- Runs a full episode for each task and prints results.
5
- Usage: python test_integration.py
6
- """
7
- import environment as env
8
- from models import Action
9
-
10
-
11
- def test_task(task_id: str) -> bool:
12
- """Run a full episode for a task. Returns True if passed."""
13
- print(f"\n{'='*60}")
14
- print(f"Testing {task_id}")
15
- print(f"{'='*60}")
16
-
17
- # Reset
18
- print("1. reset()...")
19
- obs = env.reset(task_id, ticket_index=0)
20
- episode_id = obs.episode_id
21
- print(f" [OK] episode_id={episode_id[:8]}...")
22
- print(f" ticket_id={obs.ticket.ticket_id} subject={obs.ticket.subject[:50]}")
23
- print(f" max_steps={obs.max_steps} hint={obs.hint}")
24
-
25
- # Take a relevant action
26
- print("2. step() with task action...")
27
- if task_id == "task1":
28
- action = Action(action_type="classify", category="billing", priority="high")
29
- elif task_id == "task2":
30
- action = Action(
31
- action_type="extract",
32
- extracted_entities={"customer_name": "Robert Chen", "account_id": "ACC-78234"},
33
- required_actions=["issue_refund"],
34
- )
35
- else: # task3
36
- action = Action(
37
- action_type="respond",
38
- response_text=(
39
- "We sincerely apologize for the inconvenience with your password reset. "
40
- "We will manually reset your password and send a new email immediately. "
41
- "Please check your spam folder and whitelist our domain. "
42
- "We will resolve this within the next 30 minutes."
43
- ),
44
- resolution_steps=[
45
- "verify_email_delivery",
46
- "check_spam_filters",
47
- "manual_password_reset",
48
- "follow_up_confirmation",
49
- ],
50
- )
51
-
52
- result = env.step(episode_id, action)
53
- print(f" [OK] step_reward={result.reward.step_reward:+.4f} done={result.done}")
54
-
55
- # Submit
56
- print("3. step() submit...")
57
- if not result.done:
58
- result = env.step(episode_id, Action(action_type="submit"))
59
- print(f" [OK] done={result.done} total_reward={result.reward.total_reward:.4f}")
60
-
61
- # State
62
- print("4. get_state()...")
63
- state = env.get_state(episode_id)
64
- print(f" [OK] steps={state.step_number} history_len={len(state.history)}")
65
-
66
- # Grade
67
- print("5. grade()...")
68
- score, breakdown, feedback = env.grade(episode_id)
69
- print(f" [OK] score={score:.4f}/1.0")
70
- print(f" breakdown: {', '.join(f'{k}={v:.2f}' for k, v in breakdown.items())}")
71
- print(f" feedback: {feedback}")
72
-
73
- passed = score >= 0.0 # just verify pipeline works
74
- return passed
75
-
76
-
77
- def main():
78
- print("SupportEnv Integration Test")
79
- print("=" * 60)
80
-
81
- results = []
82
- for task_id in ["task1", "task2", "task3"]:
83
- try:
84
- ok = test_task(task_id)
85
- results.append((task_id, ok, None))
86
- except Exception as exc:
87
- import traceback
88
- traceback.print_exc()
89
- results.append((task_id, False, str(exc)))
90
- finally:
91
- env._EPISODES.clear()
92
-
93
- print(f"\n{'='*60}")
94
- print("SUMMARY")
95
- print("=" * 60)
96
- all_ok = True
97
- for task_id, ok, err in results:
98
- status = "[PASS]" if ok else "[FAIL]"
99
- print(f" {status} {task_id}" + (f" — {err}" if err else ""))
100
- if not ok:
101
- all_ok = False
102
-
103
- if all_ok:
104
- print("\n[OK] All integration tests passed!")
105
- else:
106
- print("\n[FAIL] Some tests failed.")
107
- return 0 if all_ok else 1
108
-
109
-
110
- if __name__ == "__main__":
111
- import sys
112
- sys.exit(main())
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
tests_new.py DELETED
@@ -1,259 +0,0 @@
1
- """
2
- Comprehensive tests for SupportEnv.
3
-
4
- Run with: pytest tests_new.py -v
5
- """
6
- import pytest
7
-
8
- import environment as env
9
- from models import Action, Observation, StepResult, State
10
- from data import TASK_META
11
-
12
-
13
- class TestReset:
14
- """Test episode reset functionality."""
15
-
16
- def test_reset_task1(self):
17
- obs = env.reset("task1")
18
- assert isinstance(obs, Observation)
19
- assert obs.task_id == "task1"
20
- assert obs.episode_id is not None
21
- assert obs.ticket is not None
22
- assert obs.step_number == 0
23
- assert obs.max_steps == TASK_META["task1"]["max_steps"]
24
-
25
- def test_reset_task2(self):
26
- obs = env.reset("task2")
27
- assert obs.task_id == "task2"
28
- assert obs.ticket.ticket_id.startswith("T2-")
29
-
30
- def test_reset_task3(self):
31
- obs = env.reset("task3")
32
- assert obs.task_id == "task3"
33
- assert obs.ticket.ticket_id.startswith("T3-")
34
-
35
- def test_reset_with_ticket_index(self):
36
- obs = env.reset("task1", ticket_index=2)
37
- assert obs.ticket.ticket_id == "T1-003"
38
-
39
- def test_reset_invalid_task(self):
40
- with pytest.raises(ValueError):
41
- env.reset("task_unknown")
42
-
43
- def test_reset_invalid_ticket_index(self):
44
- with pytest.raises(ValueError):
45
- env.reset("task1", ticket_index=99)
46
-
47
- def test_reset_creates_episode(self):
48
- obs = env.reset("task1")
49
- assert obs.episode_id in env._EPISODES
50
- ep = env._EPISODES[obs.episode_id]
51
- assert ep["task_id"] == "task1"
52
- assert ep["step_number"] == 0
53
- assert ep["done"] is False
54
-
55
- def test_reset_hint_on_first_step(self):
56
- obs = env.reset("task1")
57
- assert obs.hint is not None
58
-
59
-
60
- class TestStep:
61
- """Test step execution."""
62
-
63
- def test_step_classify(self):
64
- obs = env.reset("task1")
65
- action = Action(action_type="classify", category="billing", priority="high")
66
- result = env.step(obs.episode_id, action)
67
- assert isinstance(result, StepResult)
68
- assert result.observation.step_number == 1
69
- assert result.reward.step_reward is not None
70
-
71
- def test_step_extract(self):
72
- obs = env.reset("task2")
73
- action = Action(
74
- action_type="extract",
75
- extracted_entities={"customer_name": "Alice"},
76
- required_actions=["issue_refund"],
77
- )
78
- result = env.step(obs.episode_id, action)
79
- assert result.observation.step_number == 1
80
-
81
- def test_step_respond(self):
82
- obs = env.reset("task3")
83
- action = Action(
84
- action_type="respond",
85
- response_text="Thank you for reaching out. We sincerely apologize for the inconvenience and will resolve this immediately.",
86
- resolution_steps=["verify_account", "issue_refund"],
87
- )
88
- result = env.step(obs.episode_id, action)
89
- assert result.observation.step_number == 1
90
-
91
- def test_step_submit_marks_done(self):
92
- obs = env.reset("task1")
93
- result = env.step(obs.episode_id, Action(action_type="submit"))
94
- assert result.done is True
95
- assert result.observation.available_actions == []
96
-
97
- def test_step_invalid_episode(self):
98
- with pytest.raises(KeyError):
99
- env.step("nonexistent-id", Action(action_type="submit"))
100
-
101
- def test_step_after_done_raises(self):
102
- obs = env.reset("task1")
103
- env.step(obs.episode_id, Action(action_type="submit"))
104
- with pytest.raises(ValueError):
105
- env.step(obs.episode_id, Action(action_type="submit"))
106
-
107
- def test_step_max_steps_ends_episode(self):
108
- obs = env.reset("task1")
109
- max_steps = obs.max_steps
110
- for i in range(max_steps):
111
- action = Action(action_type="classify", category="general", priority="low")
112
- result = env.step(obs.episode_id, action)
113
- assert result.done is True
114
-
115
- def test_thread_history_grows(self):
116
- obs = env.reset("task1")
117
- env.step(obs.episode_id, Action(action_type="classify", category="billing", priority="high"))
118
- result = env.step(obs.episode_id, Action(action_type="submit"))
119
- assert len(result.observation.thread_history) == 2
120
-
121
-
122
- class TestState:
123
- """Test state retrieval."""
124
-
125
- def test_get_state_initial(self):
126
- obs = env.reset("task1")
127
- state = env.get_state(obs.episode_id)
128
- assert isinstance(state, State)
129
- assert state.episode_id == obs.episode_id
130
- assert state.task_id == "task1"
131
- assert state.step_number == 0
132
- assert state.done is False
133
-
134
- def test_get_state_invalid(self):
135
- with pytest.raises(KeyError):
136
- env.get_state("bad-id")
137
-
138
- def test_state_history_after_step(self):
139
- obs = env.reset("task2")
140
- env.step(obs.episode_id, Action(action_type="extract", extracted_entities={}, required_actions=[]))
141
- state = env.get_state(obs.episode_id)
142
- assert len(state.history) == 1
143
- assert state.history[0]["action_type"] == "extract"
144
-
145
-
146
- class TestGraders:
147
- """Test grading for each task."""
148
-
149
- def test_grade_task1_perfect(self):
150
- """Correct category + priority on ticket 0 (billing/high)."""
151
- obs = env.reset("task1", ticket_index=0)
152
- env.step(obs.episode_id, Action(action_type="classify", category="billing", priority="high"))
153
- env.step(obs.episode_id, Action(action_type="submit"))
154
- score, breakdown, feedback = env.grade(obs.episode_id)
155
- assert score >= 0.9
156
- assert breakdown["category_correct"] == 0.50
157
- assert breakdown["priority_correct"] == 0.40
158
-
159
- def test_grade_task1_wrong_category(self):
160
- obs = env.reset("task1", ticket_index=0)
161
- env.step(obs.episode_id, Action(action_type="classify", category="technical", priority="high"))
162
- env.step(obs.episode_id, Action(action_type="submit"))
163
- score, breakdown, _ = env.grade(obs.episode_id)
164
- assert breakdown["category_correct"] == 0.0
165
- assert breakdown["priority_correct"] == 0.40
166
-
167
- def test_grade_task1_no_classify_action(self):
168
- obs = env.reset("task1")
169
- env.step(obs.episode_id, Action(action_type="submit"))
170
- score, _, _ = env.grade(obs.episode_id)
171
- assert score == 0.0
172
-
173
- def test_grade_task2_entities(self):
174
- obs = env.reset("task2", ticket_index=0)
175
- env.step(obs.episode_id, Action(
176
- action_type="extract",
177
- extracted_entities={
178
- "customer_name": "Robert Chen",
179
- "account_id": "ACC-78234",
180
- "invoice_number": "INV-20240312",
181
- "incorrect_amount": "199.00",
182
- "correct_amount": "99.00",
183
- "refund_amount": "100.00",
184
- },
185
- required_actions=["issue_refund", "send_corrected_invoice"],
186
- ))
187
- env.step(obs.episode_id, Action(action_type="submit"))
188
- score, breakdown, _ = env.grade(obs.episode_id)
189
- assert breakdown["entity_coverage"] == pytest.approx(0.60, abs=0.01)
190
- assert breakdown["action_coverage"] == pytest.approx(0.30, abs=0.01)
191
-
192
- def test_grade_task3_keywords_and_steps(self):
193
- obs = env.reset("task3", ticket_index=0)
194
- env.step(obs.episode_id, Action(
195
- action_type="respond",
196
- response_text=(
197
- "We sincerely apologize for the password reset issue. "
198
- "We will send a new reset email and ask you to check your spam folder "
199
- "and whitelist our domain. We will have this resolved within the hour."
200
- ),
201
- resolution_steps=[
202
- "verify_email_delivery",
203
- "check_spam_filters",
204
- "manual_password_reset",
205
- "follow_up_confirmation",
206
- ],
207
- ))
208
- env.step(obs.episode_id, Action(action_type="submit"))
209
- score, breakdown, _ = env.grade(obs.episode_id)
210
- assert score >= 0.7
211
- assert breakdown["length_adequate"] == 0.10
212
- assert breakdown["no_empty_steps"] == 0.05
213
-
214
- def test_grade_not_done_raises(self):
215
- obs = env.reset("task1")
216
- with pytest.raises(ValueError):
217
- env.grade(obs.episode_id)
218
-
219
- def test_grade_invalid_episode_raises(self):
220
- with pytest.raises(KeyError):
221
- env.grade("bad-id")
222
-
223
- def test_score_in_range(self):
224
- for task_id in ["task1", "task2", "task3"]:
225
- obs = env.reset(task_id)
226
- env.step(obs.episode_id, Action(action_type="submit"))
227
- score, _, _ = env.grade(obs.episode_id)
228
- assert 0.0 <= score <= 1.0
229
-
230
-
231
- class TestRewards:
232
- """Test reward signals."""
233
-
234
- def test_step_reward_is_float(self):
235
- obs = env.reset("task1")
236
- result = env.step(obs.episode_id, Action(action_type="classify", category="billing", priority="high"))
237
- assert isinstance(result.reward.step_reward, float)
238
-
239
- def test_total_reward_accumulates(self):
240
- obs = env.reset("task2")
241
- r1 = env.step(obs.episode_id, Action(action_type="extract", extracted_entities={}, required_actions=[]))
242
- r2 = env.step(obs.episode_id, Action(action_type="submit"))
243
- assert r2.reward.total_reward != r1.reward.total_reward
244
-
245
- def test_submit_bonus_applied(self):
246
- obs = env.reset("task1")
247
- result = env.step(obs.episode_id, Action(action_type="submit"))
248
- # submit_bonus=0.05 minus step_cost=0.02 = +0.03 base before grader
249
- assert result.reward.step_reward > 0.0
250
-
251
-
252
- @pytest.fixture(autouse=True)
253
- def cleanup():
254
- yield
255
- env._EPISODES.clear()
256
-
257
-
258
- if __name__ == "__main__":
259
- pytest.main([__file__, "-v"])