strict changes for 0 and 1 value issue
Browse files- .gitignore +3 -1
- README.md +3 -1
- inference.py +17 -6
- setup.md +3 -3
.gitignore
CHANGED
|
@@ -9,4 +9,6 @@ dist/
|
|
| 9 |
build/
|
| 10 |
.pytest_cache/
|
| 11 |
myenv/
|
| 12 |
-
res/
|
|
|
|
|
|
|
|
|
| 9 |
build/
|
| 10 |
.pytest_cache/
|
| 11 |
myenv/
|
| 12 |
+
res/
|
| 13 |
+
.claude
|
| 14 |
+
.codex
|
README.md
CHANGED
|
@@ -153,7 +153,9 @@ The baseline emits strict structured stdout lines only:
|
|
| 153 |
|
| 154 |
- `[START] task=<...> env=<...> model=<...>`
|
| 155 |
- `[STEP] step=<...> action=<...> reward=<...> done=<...> error=<...>`
|
| 156 |
-
- `[END] success=<...> steps=<...> rewards=<...>`
|
|
|
|
|
|
|
| 157 |
|
| 158 |
## Docker
|
| 159 |
|
|
|
|
| 153 |
|
| 154 |
- `[START] task=<...> env=<...> model=<...>`
|
| 155 |
- `[STEP] step=<...> action=<...> reward=<...> done=<...> error=<...>`
|
| 156 |
+
- `[END] success=<...> steps=<...> score=<...> rewards=<...>`
|
| 157 |
+
|
| 158 |
+
Validator note: each emitted task `score` must stay strictly inside `(0, 1)`.
|
| 159 |
|
| 160 |
## Docker
|
| 161 |
|
inference.py
CHANGED
|
@@ -36,6 +36,7 @@ ENV_BASE_URL = (
|
|
| 36 |
TEMPERATURE = 0.3
|
| 37 |
MAX_TOKENS = 1024
|
| 38 |
BENCHMARK = "supportenv"
|
|
|
|
| 39 |
|
| 40 |
TASKS = [
|
| 41 |
{"task_id": "task1", "name": "Ticket Classification", "tickets": 5},
|
|
@@ -61,14 +62,21 @@ def log_step(step: int, action: str, reward: float, done: bool, error: Optional[
|
|
| 61 |
)
|
| 62 |
|
| 63 |
|
| 64 |
-
def log_end(success: bool, steps: int, rewards: List[float]) -> None:
|
| 65 |
rewards_str = ",".join(f"{r:.2f}" for r in rewards)
|
| 66 |
print(
|
| 67 |
-
f"[END] success={str(success).lower()} steps={steps} rewards={rewards_str}",
|
| 68 |
flush=True,
|
| 69 |
)
|
| 70 |
|
| 71 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 72 |
# ---------------------------------------------------------------------------
|
| 73 |
# Environment HTTP helpers
|
| 74 |
# ---------------------------------------------------------------------------
|
|
@@ -192,6 +200,7 @@ def run_episode(
|
|
| 192 |
steps_taken = 0
|
| 193 |
success = False
|
| 194 |
error_msg: Optional[str] = None
|
|
|
|
| 195 |
|
| 196 |
try:
|
| 197 |
# Reset
|
|
@@ -228,21 +237,23 @@ def run_episode(
|
|
| 228 |
# Grade
|
| 229 |
grade = env_request("POST", "/grader", json={"episode_id": episode_id})
|
| 230 |
final_score = grade["score"]
|
| 231 |
-
success = final_score >= 0.5
|
| 232 |
|
| 233 |
except Exception as exc:
|
| 234 |
error_msg = str(exc)
|
| 235 |
print(f"[DEBUG] Episode error: {exc}", file=sys.stderr, flush=True)
|
| 236 |
|
| 237 |
-
#
|
| 238 |
-
task_score =
|
| 239 |
-
|
|
|
|
| 240 |
|
| 241 |
return {
|
| 242 |
"task_id": task_id,
|
| 243 |
"ticket_index": ticket_index,
|
| 244 |
"steps": steps_taken,
|
| 245 |
"rewards": rewards,
|
|
|
|
| 246 |
"success": success,
|
| 247 |
}
|
| 248 |
|
|
|
|
| 36 |
TEMPERATURE = 0.3
|
| 37 |
MAX_TOKENS = 1024
|
| 38 |
BENCHMARK = "supportenv"
|
| 39 |
+
SCORE_EPSILON = 0.01
|
| 40 |
|
| 41 |
TASKS = [
|
| 42 |
{"task_id": "task1", "name": "Ticket Classification", "tickets": 5},
|
|
|
|
| 62 |
)
|
| 63 |
|
| 64 |
|
| 65 |
+
def log_end(success: bool, steps: int, score: float, rewards: List[float]) -> None:
|
| 66 |
rewards_str = ",".join(f"{r:.2f}" for r in rewards)
|
| 67 |
print(
|
| 68 |
+
f"[END] success={str(success).lower()} steps={steps} score={score:.2f} rewards={rewards_str}",
|
| 69 |
flush=True,
|
| 70 |
)
|
| 71 |
|
| 72 |
|
| 73 |
+
def _strict_open_score(value: Optional[float]) -> float:
|
| 74 |
+
"""Keep emitted task scores inside the validator's required open interval."""
|
| 75 |
+
if value is None:
|
| 76 |
+
value = SCORE_EPSILON
|
| 77 |
+
return round(min(max(float(value), SCORE_EPSILON), 1.0 - SCORE_EPSILON), 4)
|
| 78 |
+
|
| 79 |
+
|
| 80 |
# ---------------------------------------------------------------------------
|
| 81 |
# Environment HTTP helpers
|
| 82 |
# ---------------------------------------------------------------------------
|
|
|
|
| 200 |
steps_taken = 0
|
| 201 |
success = False
|
| 202 |
error_msg: Optional[str] = None
|
| 203 |
+
final_score = SCORE_EPSILON
|
| 204 |
|
| 205 |
try:
|
| 206 |
# Reset
|
|
|
|
| 237 |
# Grade
|
| 238 |
grade = env_request("POST", "/grader", json={"episode_id": episode_id})
|
| 239 |
final_score = grade["score"]
|
| 240 |
+
success = _strict_open_score(final_score) >= 0.5
|
| 241 |
|
| 242 |
except Exception as exc:
|
| 243 |
error_msg = str(exc)
|
| 244 |
print(f"[DEBUG] Episode error: {exc}", file=sys.stderr, flush=True)
|
| 245 |
|
| 246 |
+
# Emit the canonical task score separately from dense per-step rewards.
|
| 247 |
+
task_score = _strict_open_score(final_score)
|
| 248 |
+
logged_rewards = [_strict_open_score(r) for r in (rewards or [task_score])]
|
| 249 |
+
log_end(success=success, steps=steps_taken, score=task_score, rewards=logged_rewards)
|
| 250 |
|
| 251 |
return {
|
| 252 |
"task_id": task_id,
|
| 253 |
"ticket_index": ticket_index,
|
| 254 |
"steps": steps_taken,
|
| 255 |
"rewards": rewards,
|
| 256 |
+
"score": task_score,
|
| 257 |
"success": success,
|
| 258 |
}
|
| 259 |
|
setup.md
CHANGED
|
@@ -15,7 +15,7 @@ Most checks align to this flow:
|
|
| 15 |
|
| 16 |
- `app.py`: FastAPI API surface (`/reset`, `/step`, `/state`, `/tasks`, `/grader`, `/health`)
|
| 17 |
- `environment.py`: episode lifecycle and reward accumulation (`reset`, `step`, `get_state`, `grade`)
|
| 18 |
-
- `graders.py`: deterministic terminal scoring per task with score clamped to `
|
| 19 |
- `data.py`: task metadata and ticket datasets with ground truth labels/entities/steps
|
| 20 |
- `models.py`: typed Pydantic models used by API and internal state
|
| 21 |
- `inference.py`: baseline runner; calls the API, logs strict `[START]/[STEP]/[END]`
|
|
@@ -105,7 +105,7 @@ python inference.py
|
|
| 105 |
|
| 106 |
## 7. Example scorer sanity checks
|
| 107 |
|
| 108 |
-
- Task 1: submit `classify` then `submit`, verify non-binary reward and final score in `
|
| 109 |
- Task 2: include deterministic entity/action coverage keys from ticket text
|
| 110 |
- Task 3: include professional response plus ordered resolution steps
|
| 111 |
|
|
@@ -116,4 +116,4 @@ python inference.py
|
|
| 116 |
- Missing script entrypoint (`server = server.app:main`)
|
| 117 |
- App not serving on `0.0.0.0:7860`
|
| 118 |
- Duplicate HF variable/secret names in Space settings
|
| 119 |
-
- Invalid or missing `HF_TOKEN` for real LLM inference
|
|
|
|
| 15 |
|
| 16 |
- `app.py`: FastAPI API surface (`/reset`, `/step`, `/state`, `/tasks`, `/grader`, `/health`)
|
| 17 |
- `environment.py`: episode lifecycle and reward accumulation (`reset`, `step`, `get_state`, `grade`)
|
| 18 |
+
- `graders.py`: deterministic terminal scoring per task with score clamped to `(0, 1)`
|
| 19 |
- `data.py`: task metadata and ticket datasets with ground truth labels/entities/steps
|
| 20 |
- `models.py`: typed Pydantic models used by API and internal state
|
| 21 |
- `inference.py`: baseline runner; calls the API, logs strict `[START]/[STEP]/[END]`
|
|
|
|
| 105 |
|
| 106 |
## 7. Example scorer sanity checks
|
| 107 |
|
| 108 |
+
- Task 1: submit `classify` then `submit`, verify non-binary reward and final score in `(0, 1)`
|
| 109 |
- Task 2: include deterministic entity/action coverage keys from ticket text
|
| 110 |
- Task 3: include professional response plus ordered resolution steps
|
| 111 |
|
|
|
|
| 116 |
- Missing script entrypoint (`server = server.app:main`)
|
| 117 |
- App not serving on `0.0.0.0:7860`
|
| 118 |
- Duplicate HF variable/secret names in Space settings
|
| 119 |
+
- Invalid or missing `HF_TOKEN` for real LLM inference
|