Shivoo29 commited on
Commit
ae76d2c
·
1 Parent(s): 5ac473e

strict changes for 0 and 1 value issue

Browse files
Files changed (4) hide show
  1. .gitignore +3 -1
  2. README.md +3 -1
  3. inference.py +17 -6
  4. setup.md +3 -3
.gitignore CHANGED
@@ -9,4 +9,6 @@ dist/
9
  build/
10
  .pytest_cache/
11
  myenv/
12
- res/
 
 
 
9
  build/
10
  .pytest_cache/
11
  myenv/
12
+ res/
13
+ .claude
14
+ .codex
README.md CHANGED
@@ -153,7 +153,9 @@ The baseline emits strict structured stdout lines only:
153
 
154
  - `[START] task=<...> env=<...> model=<...>`
155
  - `[STEP] step=<...> action=<...> reward=<...> done=<...> error=<...>`
156
- - `[END] success=<...> steps=<...> rewards=<...>`
 
 
157
 
158
  ## Docker
159
 
 
153
 
154
  - `[START] task=<...> env=<...> model=<...>`
155
  - `[STEP] step=<...> action=<...> reward=<...> done=<...> error=<...>`
156
+ - `[END] success=<...> steps=<...> score=<...> rewards=<...>`
157
+
158
+ Validator note: each emitted task `score` must stay strictly inside `(0, 1)`.
159
 
160
  ## Docker
161
 
inference.py CHANGED
@@ -36,6 +36,7 @@ ENV_BASE_URL = (
36
  TEMPERATURE = 0.3
37
  MAX_TOKENS = 1024
38
  BENCHMARK = "supportenv"
 
39
 
40
  TASKS = [
41
  {"task_id": "task1", "name": "Ticket Classification", "tickets": 5},
@@ -61,14 +62,21 @@ def log_step(step: int, action: str, reward: float, done: bool, error: Optional[
61
  )
62
 
63
 
64
- def log_end(success: bool, steps: int, rewards: List[float]) -> None:
65
  rewards_str = ",".join(f"{r:.2f}" for r in rewards)
66
  print(
67
- f"[END] success={str(success).lower()} steps={steps} rewards={rewards_str}",
68
  flush=True,
69
  )
70
 
71
 
 
 
 
 
 
 
 
72
  # ---------------------------------------------------------------------------
73
  # Environment HTTP helpers
74
  # ---------------------------------------------------------------------------
@@ -192,6 +200,7 @@ def run_episode(
192
  steps_taken = 0
193
  success = False
194
  error_msg: Optional[str] = None
 
195
 
196
  try:
197
  # Reset
@@ -228,21 +237,23 @@ def run_episode(
228
  # Grade
229
  grade = env_request("POST", "/grader", json={"episode_id": episode_id})
230
  final_score = grade["score"]
231
- success = final_score >= 0.5
232
 
233
  except Exception as exc:
234
  error_msg = str(exc)
235
  print(f"[DEBUG] Episode error: {exc}", file=sys.stderr, flush=True)
236
 
237
- # Log grader score as the canonical task score (always in (0,1))
238
- task_score = round(min(max(final_score if final_score > 0 else 0.01, 0.01), 0.99), 4)
239
- log_end(success=success, steps=steps_taken, rewards=[task_score])
 
240
 
241
  return {
242
  "task_id": task_id,
243
  "ticket_index": ticket_index,
244
  "steps": steps_taken,
245
  "rewards": rewards,
 
246
  "success": success,
247
  }
248
 
 
36
  TEMPERATURE = 0.3
37
  MAX_TOKENS = 1024
38
  BENCHMARK = "supportenv"
39
+ SCORE_EPSILON = 0.01
40
 
41
  TASKS = [
42
  {"task_id": "task1", "name": "Ticket Classification", "tickets": 5},
 
62
  )
63
 
64
 
65
+ def log_end(success: bool, steps: int, score: float, rewards: List[float]) -> None:
66
  rewards_str = ",".join(f"{r:.2f}" for r in rewards)
67
  print(
68
+ f"[END] success={str(success).lower()} steps={steps} score={score:.2f} rewards={rewards_str}",
69
  flush=True,
70
  )
71
 
72
 
73
+ def _strict_open_score(value: Optional[float]) -> float:
74
+ """Keep emitted task scores inside the validator's required open interval."""
75
+ if value is None:
76
+ value = SCORE_EPSILON
77
+ return round(min(max(float(value), SCORE_EPSILON), 1.0 - SCORE_EPSILON), 4)
78
+
79
+
80
  # ---------------------------------------------------------------------------
81
  # Environment HTTP helpers
82
  # ---------------------------------------------------------------------------
 
200
  steps_taken = 0
201
  success = False
202
  error_msg: Optional[str] = None
203
+ final_score = SCORE_EPSILON
204
 
205
  try:
206
  # Reset
 
237
  # Grade
238
  grade = env_request("POST", "/grader", json={"episode_id": episode_id})
239
  final_score = grade["score"]
240
+ success = _strict_open_score(final_score) >= 0.5
241
 
242
  except Exception as exc:
243
  error_msg = str(exc)
244
  print(f"[DEBUG] Episode error: {exc}", file=sys.stderr, flush=True)
245
 
246
+ # Emit the canonical task score separately from dense per-step rewards.
247
+ task_score = _strict_open_score(final_score)
248
+ logged_rewards = [_strict_open_score(r) for r in (rewards or [task_score])]
249
+ log_end(success=success, steps=steps_taken, score=task_score, rewards=logged_rewards)
250
 
251
  return {
252
  "task_id": task_id,
253
  "ticket_index": ticket_index,
254
  "steps": steps_taken,
255
  "rewards": rewards,
256
+ "score": task_score,
257
  "success": success,
258
  }
259
 
setup.md CHANGED
@@ -15,7 +15,7 @@ Most checks align to this flow:
15
 
16
  - `app.py`: FastAPI API surface (`/reset`, `/step`, `/state`, `/tasks`, `/grader`, `/health`)
17
  - `environment.py`: episode lifecycle and reward accumulation (`reset`, `step`, `get_state`, `grade`)
18
- - `graders.py`: deterministic terminal scoring per task with score clamped to `[0.0, 1.0]`
19
  - `data.py`: task metadata and ticket datasets with ground truth labels/entities/steps
20
  - `models.py`: typed Pydantic models used by API and internal state
21
  - `inference.py`: baseline runner; calls the API, logs strict `[START]/[STEP]/[END]`
@@ -105,7 +105,7 @@ python inference.py
105
 
106
  ## 7. Example scorer sanity checks
107
 
108
- - Task 1: submit `classify` then `submit`, verify non-binary reward and final score in `[0, 1]`
109
  - Task 2: include deterministic entity/action coverage keys from ticket text
110
  - Task 3: include professional response plus ordered resolution steps
111
 
@@ -116,4 +116,4 @@ python inference.py
116
  - Missing script entrypoint (`server = server.app:main`)
117
  - App not serving on `0.0.0.0:7860`
118
  - Duplicate HF variable/secret names in Space settings
119
- - Invalid or missing `HF_TOKEN` for real LLM inference
 
15
 
16
  - `app.py`: FastAPI API surface (`/reset`, `/step`, `/state`, `/tasks`, `/grader`, `/health`)
17
  - `environment.py`: episode lifecycle and reward accumulation (`reset`, `step`, `get_state`, `grade`)
18
+ - `graders.py`: deterministic terminal scoring per task with score clamped to `(0, 1)`
19
  - `data.py`: task metadata and ticket datasets with ground truth labels/entities/steps
20
  - `models.py`: typed Pydantic models used by API and internal state
21
  - `inference.py`: baseline runner; calls the API, logs strict `[START]/[STEP]/[END]`
 
105
 
106
  ## 7. Example scorer sanity checks
107
 
108
+ - Task 1: submit `classify` then `submit`, verify non-binary reward and final score in `(0, 1)`
109
  - Task 2: include deterministic entity/action coverage keys from ticket text
110
  - Task 3: include professional response plus ordered resolution steps
111
 
 
116
  - Missing script entrypoint (`server = server.app:main`)
117
  - App not serving on `0.0.0.0:7860`
118
  - Duplicate HF variable/secret names in Space settings
119
+ - Invalid or missing `HF_TOKEN` for real LLM inference