Spaces:
Sleeping
Sleeping
Upload support triage environment
Browse files- README.md +2 -1
- inference.py +7 -14
- support_triage_env/client.py +0 -6
- support_triage_env/environment.py +0 -9
- support_triage_env/models.py +0 -8
- tests/test_environment.py +5 -1
README.md
CHANGED
|
@@ -26,6 +26,8 @@ This environment models a real operations workflow that human support leads and
|
|
| 26 |
|
| 27 |
The key design goal is to punish shallow “form filling.” An agent can no longer get a high score just by memorizing queue and status values. It must look up order, account, policy, or fraud signals first, and the grader rewards or penalizes that investigation path directly.
|
| 28 |
|
|
|
|
|
|
|
| 29 |
## Task families
|
| 30 |
|
| 31 |
The environment ships with three task families and seeded procedural variants inside each family:
|
|
@@ -82,7 +84,6 @@ Each observation includes:
|
|
| 82 |
- the current structured workspace
|
| 83 |
- lookup targets for policy, order, account, and fraud actions
|
| 84 |
- valid actions and allowed values
|
| 85 |
-
- progress breakdown and aggregate completion score
|
| 86 |
- `last_action_error`
|
| 87 |
- `steps_remaining`
|
| 88 |
|
|
|
|
| 26 |
|
| 27 |
The key design goal is to punish shallow “form filling.” An agent can no longer get a high score just by memorizing queue and status values. It must look up order, account, policy, or fraud signals first, and the grader rewards or penalizes that investigation path directly.
|
| 28 |
|
| 29 |
+
The public observation intentionally omits exact grader targets such as per-dimension score breakdowns and literal reply/note requirement slots. The agent sees the task, policies, backend evidence, and workspace state, then has to infer the correct action from that evidence.
|
| 30 |
+
|
| 31 |
## Task families
|
| 32 |
|
| 33 |
The environment ships with three task families and seeded procedural variants inside each family:
|
|
|
|
| 84 |
- the current structured workspace
|
| 85 |
- lookup targets for policy, order, account, and fraud actions
|
| 86 |
- valid actions and allowed values
|
|
|
|
| 87 |
- `last_action_error`
|
| 88 |
- `steps_remaining`
|
| 89 |
|
inference.py
CHANGED
|
@@ -97,9 +97,8 @@ def _normalize_tag(value: str | None) -> str:
|
|
| 97 |
return normalized
|
| 98 |
|
| 99 |
|
| 100 |
-
def
|
| 101 |
-
|
| 102 |
-
return all(_normalize_text(requirement) in normalized for requirement in requirements)
|
| 103 |
|
| 104 |
|
| 105 |
def _action_for_log(action: TriageAction) -> str:
|
|
@@ -133,20 +132,15 @@ def _build_prompt_with_history(observation, recent_steps: list[dict[str, str]])
|
|
| 133 |
"ticket": observation.ticket.model_dump(),
|
| 134 |
"policy_notes": observation.policy_notes,
|
| 135 |
"investigation_requirements": observation.investigation_requirements,
|
| 136 |
-
"note_requirements": observation.note_requirements,
|
| 137 |
-
"reply_requirements": observation.reply_requirements,
|
| 138 |
"investigation_log": [record.model_dump() for record in observation.investigation_log],
|
| 139 |
"workspace": observation.workspace.model_dump(),
|
| 140 |
"allowed_values": observation.allowed_values,
|
| 141 |
"available_actions": observation.available_actions,
|
| 142 |
-
"completion_score": observation.completion_score,
|
| 143 |
-
"missing_items": observation.missing_items,
|
| 144 |
"last_action_error": observation.last_action_error,
|
| 145 |
"steps_remaining": observation.steps_remaining,
|
| 146 |
"recent_steps": recent_steps[-5:],
|
| 147 |
}
|
| 148 |
|
| 149 |
-
solved_items = [key for key, score in observation.score_breakdown.items() if score >= 0.999]
|
| 150 |
avoid_pairs = [
|
| 151 |
step["action"]
|
| 152 |
for step in recent_steps[-5:]
|
|
@@ -157,9 +151,8 @@ def _build_prompt_with_history(observation, recent_steps: list[dict[str, str]])
|
|
| 157 |
"Look up missing evidence before finalizing the workflow.\n"
|
| 158 |
"Choose exactly one next action with the choose_action function.\n"
|
| 159 |
"Set value to null only for submit.\n"
|
| 160 |
-
f"Already solved areas: {solved_items}.\n"
|
| 161 |
f"Do not repeat recently bad actions: {avoid_pairs}.\n"
|
| 162 |
-
"Prefer
|
| 163 |
f"Observation:{json.dumps(payload, separators=(',', ':'))}"
|
| 164 |
)
|
| 165 |
|
|
@@ -411,10 +404,10 @@ def _planned_action(observation) -> TriageAction:
|
|
| 411 |
if _normalize_tag(tag) not in existing_tags:
|
| 412 |
return TriageAction(action_type="add_tag", value=tag)
|
| 413 |
|
| 414 |
-
if not
|
| 415 |
return TriageAction(action_type="add_internal_note", value=plan["note"])
|
| 416 |
|
| 417 |
-
if not
|
| 418 |
return TriageAction(action_type="draft_reply", value=plan["reply"])
|
| 419 |
|
| 420 |
return TriageAction(action_type="submit")
|
|
@@ -439,10 +432,10 @@ def _prefer_model_action(observation, candidate: TriageAction, planned: TriageAc
|
|
| 439 |
return _normalize_tag(candidate.value) == _normalize_tag(planned.value)
|
| 440 |
|
| 441 |
if candidate.action_type == "add_internal_note":
|
| 442 |
-
return
|
| 443 |
|
| 444 |
if candidate.action_type == "draft_reply":
|
| 445 |
-
return
|
| 446 |
|
| 447 |
return candidate.action_type == "submit"
|
| 448 |
|
|
|
|
| 97 |
return normalized
|
| 98 |
|
| 99 |
|
| 100 |
+
def _matches_expected_text(value: str | None, expected: str) -> bool:
|
| 101 |
+
return _normalize_text(value) == _normalize_text(expected)
|
|
|
|
| 102 |
|
| 103 |
|
| 104 |
def _action_for_log(action: TriageAction) -> str:
|
|
|
|
| 132 |
"ticket": observation.ticket.model_dump(),
|
| 133 |
"policy_notes": observation.policy_notes,
|
| 134 |
"investigation_requirements": observation.investigation_requirements,
|
|
|
|
|
|
|
| 135 |
"investigation_log": [record.model_dump() for record in observation.investigation_log],
|
| 136 |
"workspace": observation.workspace.model_dump(),
|
| 137 |
"allowed_values": observation.allowed_values,
|
| 138 |
"available_actions": observation.available_actions,
|
|
|
|
|
|
|
| 139 |
"last_action_error": observation.last_action_error,
|
| 140 |
"steps_remaining": observation.steps_remaining,
|
| 141 |
"recent_steps": recent_steps[-5:],
|
| 142 |
}
|
| 143 |
|
|
|
|
| 144 |
avoid_pairs = [
|
| 145 |
step["action"]
|
| 146 |
for step in recent_steps[-5:]
|
|
|
|
| 151 |
"Look up missing evidence before finalizing the workflow.\n"
|
| 152 |
"Choose exactly one next action with the choose_action function.\n"
|
| 153 |
"Set value to null only for submit.\n"
|
|
|
|
| 154 |
f"Do not repeat recently bad actions: {avoid_pairs}.\n"
|
| 155 |
+
"Prefer unresolved investigation and policy-backed workflow decisions over guesswork.\n"
|
| 156 |
f"Observation:{json.dumps(payload, separators=(',', ':'))}"
|
| 157 |
)
|
| 158 |
|
|
|
|
| 404 |
if _normalize_tag(tag) not in existing_tags:
|
| 405 |
return TriageAction(action_type="add_tag", value=tag)
|
| 406 |
|
| 407 |
+
if not _matches_expected_text(workspace.internal_note, plan["note"]):
|
| 408 |
return TriageAction(action_type="add_internal_note", value=plan["note"])
|
| 409 |
|
| 410 |
+
if not _matches_expected_text(workspace.draft_reply, plan["reply"]):
|
| 411 |
return TriageAction(action_type="draft_reply", value=plan["reply"])
|
| 412 |
|
| 413 |
return TriageAction(action_type="submit")
|
|
|
|
| 432 |
return _normalize_tag(candidate.value) == _normalize_tag(planned.value)
|
| 433 |
|
| 434 |
if candidate.action_type == "add_internal_note":
|
| 435 |
+
return _matches_expected_text(candidate.value, planned.value or "")
|
| 436 |
|
| 437 |
if candidate.action_type == "draft_reply":
|
| 438 |
+
return _matches_expected_text(candidate.value, planned.value or "")
|
| 439 |
|
| 440 |
return candidate.action_type == "submit"
|
| 441 |
|
support_triage_env/client.py
CHANGED
|
@@ -30,8 +30,6 @@ class SupportTriageEnv(EnvClient[TriageAction, TriageObservation, TriageState]):
|
|
| 30 |
ticket=SupportTicket.model_validate(obs_data.get("ticket", {})),
|
| 31 |
policy_notes=obs_data.get("policy_notes", []),
|
| 32 |
investigation_requirements=obs_data.get("investigation_requirements", []),
|
| 33 |
-
note_requirements=obs_data.get("note_requirements", []),
|
| 34 |
-
reply_requirements=obs_data.get("reply_requirements", []),
|
| 35 |
investigation_log=[
|
| 36 |
InvestigationRecord.model_validate(item)
|
| 37 |
for item in obs_data.get("investigation_log", [])
|
|
@@ -39,9 +37,6 @@ class SupportTriageEnv(EnvClient[TriageAction, TriageObservation, TriageState]):
|
|
| 39 |
workspace=TriageWorkspace.model_validate(obs_data.get("workspace", {})),
|
| 40 |
available_actions=obs_data.get("available_actions", []),
|
| 41 |
allowed_values=obs_data.get("allowed_values", {}),
|
| 42 |
-
completion_score=obs_data.get("completion_score", 0.0),
|
| 43 |
-
score_breakdown=obs_data.get("score_breakdown", {}),
|
| 44 |
-
missing_items=obs_data.get("missing_items", []),
|
| 45 |
last_action_error=obs_data.get("last_action_error"),
|
| 46 |
steps_remaining=obs_data.get("steps_remaining", 0),
|
| 47 |
reward=payload.get("reward"),
|
|
@@ -68,6 +63,5 @@ class SupportTriageEnv(EnvClient[TriageAction, TriageObservation, TriageState]):
|
|
| 68 |
],
|
| 69 |
last_action_error=payload.get("last_action_error"),
|
| 70 |
completion_score=payload.get("completion_score", 0.0),
|
| 71 |
-
score_breakdown=payload.get("score_breakdown", {}),
|
| 72 |
submitted=payload.get("submitted", False),
|
| 73 |
)
|
|
|
|
| 30 |
ticket=SupportTicket.model_validate(obs_data.get("ticket", {})),
|
| 31 |
policy_notes=obs_data.get("policy_notes", []),
|
| 32 |
investigation_requirements=obs_data.get("investigation_requirements", []),
|
|
|
|
|
|
|
| 33 |
investigation_log=[
|
| 34 |
InvestigationRecord.model_validate(item)
|
| 35 |
for item in obs_data.get("investigation_log", [])
|
|
|
|
| 37 |
workspace=TriageWorkspace.model_validate(obs_data.get("workspace", {})),
|
| 38 |
available_actions=obs_data.get("available_actions", []),
|
| 39 |
allowed_values=obs_data.get("allowed_values", {}),
|
|
|
|
|
|
|
|
|
|
| 40 |
last_action_error=obs_data.get("last_action_error"),
|
| 41 |
steps_remaining=obs_data.get("steps_remaining", 0),
|
| 42 |
reward=payload.get("reward"),
|
|
|
|
| 63 |
],
|
| 64 |
last_action_error=payload.get("last_action_error"),
|
| 65 |
completion_score=payload.get("completion_score", 0.0),
|
|
|
|
| 66 |
submitted=payload.get("submitted", False),
|
| 67 |
)
|
support_triage_env/environment.py
CHANGED
|
@@ -60,7 +60,6 @@ class SupportTriageCoreEnvironment:
|
|
| 60 |
investigation_log=[],
|
| 61 |
last_action_error=None,
|
| 62 |
completion_score=0.0,
|
| 63 |
-
score_breakdown={},
|
| 64 |
submitted=False,
|
| 65 |
)
|
| 66 |
self._last_action_signature = None
|
|
@@ -147,8 +146,6 @@ class SupportTriageCoreEnvironment:
|
|
| 147 |
done=observation.done,
|
| 148 |
info=TriageStepInfo(
|
| 149 |
last_action_error=self._state.last_action_error,
|
| 150 |
-
score_breakdown=dict(self._state.score_breakdown),
|
| 151 |
-
missing_items=list(observation.missing_items),
|
| 152 |
submitted=self._state.submitted,
|
| 153 |
step_count=self._state.step_count,
|
| 154 |
),
|
|
@@ -226,7 +223,6 @@ class SupportTriageCoreEnvironment:
|
|
| 226 |
|
| 227 |
def _sync_grade(self, grade) -> None:
|
| 228 |
self._state.completion_score = grade.score
|
| 229 |
-
self._state.score_breakdown = grade.breakdown
|
| 230 |
|
| 231 |
def _run_lookup(self, action_type: str, target: str | None) -> bool:
|
| 232 |
source = LOOKUP_ACTIONS[action_type]
|
|
@@ -267,15 +263,10 @@ class SupportTriageCoreEnvironment:
|
|
| 267 |
ticket=self._task.ticket,
|
| 268 |
policy_notes=self._task.policy_notes,
|
| 269 |
investigation_requirements=list(self._task.investigation_requirements),
|
| 270 |
-
note_requirements=[slot.alternatives[0] for slot in self._task.note_slots],
|
| 271 |
-
reply_requirements=[slot.alternatives[0] for slot in self._task.reply_slots],
|
| 272 |
investigation_log=[record.model_copy(deep=True) for record in self._state.investigation_log],
|
| 273 |
workspace=self._state.workspace.model_copy(deep=True),
|
| 274 |
available_actions=AVAILABLE_ACTIONS,
|
| 275 |
allowed_values=allowed_values(self._task),
|
| 276 |
-
completion_score=current_grade.score,
|
| 277 |
-
score_breakdown=current_grade.breakdown,
|
| 278 |
-
missing_items=current_grade.missing_items,
|
| 279 |
last_action_error=self._state.last_action_error,
|
| 280 |
steps_remaining=max(0, MAX_STEPS - self._state.step_count),
|
| 281 |
reward=reward,
|
|
|
|
| 60 |
investigation_log=[],
|
| 61 |
last_action_error=None,
|
| 62 |
completion_score=0.0,
|
|
|
|
| 63 |
submitted=False,
|
| 64 |
)
|
| 65 |
self._last_action_signature = None
|
|
|
|
| 146 |
done=observation.done,
|
| 147 |
info=TriageStepInfo(
|
| 148 |
last_action_error=self._state.last_action_error,
|
|
|
|
|
|
|
| 149 |
submitted=self._state.submitted,
|
| 150 |
step_count=self._state.step_count,
|
| 151 |
),
|
|
|
|
| 223 |
|
| 224 |
def _sync_grade(self, grade) -> None:
|
| 225 |
self._state.completion_score = grade.score
|
|
|
|
| 226 |
|
| 227 |
def _run_lookup(self, action_type: str, target: str | None) -> bool:
|
| 228 |
source = LOOKUP_ACTIONS[action_type]
|
|
|
|
| 263 |
ticket=self._task.ticket,
|
| 264 |
policy_notes=self._task.policy_notes,
|
| 265 |
investigation_requirements=list(self._task.investigation_requirements),
|
|
|
|
|
|
|
| 266 |
investigation_log=[record.model_copy(deep=True) for record in self._state.investigation_log],
|
| 267 |
workspace=self._state.workspace.model_copy(deep=True),
|
| 268 |
available_actions=AVAILABLE_ACTIONS,
|
| 269 |
allowed_values=allowed_values(self._task),
|
|
|
|
|
|
|
|
|
|
| 270 |
last_action_error=self._state.last_action_error,
|
| 271 |
steps_remaining=max(0, MAX_STEPS - self._state.step_count),
|
| 272 |
reward=reward,
|
support_triage_env/models.py
CHANGED
|
@@ -80,15 +80,10 @@ class TriageObservation(Observation):
|
|
| 80 |
ticket: SupportTicket
|
| 81 |
policy_notes: list[str] = Field(default_factory=list)
|
| 82 |
investigation_requirements: list[str] = Field(default_factory=list)
|
| 83 |
-
note_requirements: list[str] = Field(default_factory=list)
|
| 84 |
-
reply_requirements: list[str] = Field(default_factory=list)
|
| 85 |
investigation_log: list["InvestigationRecord"] = Field(default_factory=list)
|
| 86 |
workspace: TriageWorkspace
|
| 87 |
available_actions: list[str] = Field(default_factory=list)
|
| 88 |
allowed_values: dict[str, list[str]] = Field(default_factory=dict)
|
| 89 |
-
completion_score: float = 0.0
|
| 90 |
-
score_breakdown: dict[str, float] = Field(default_factory=dict)
|
| 91 |
-
missing_items: list[str] = Field(default_factory=list)
|
| 92 |
last_action_error: str | None = None
|
| 93 |
steps_remaining: int = 0
|
| 94 |
|
|
@@ -102,7 +97,6 @@ class TriageState(State):
|
|
| 102 |
investigation_log: list["InvestigationRecord"] = Field(default_factory=list)
|
| 103 |
last_action_error: str | None = None
|
| 104 |
completion_score: float = 0.0
|
| 105 |
-
score_breakdown: dict[str, float] = Field(default_factory=dict)
|
| 106 |
submitted: bool = False
|
| 107 |
|
| 108 |
|
|
@@ -136,8 +130,6 @@ class TriageReward(BaseModel):
|
|
| 136 |
|
| 137 |
class TriageStepInfo(BaseModel):
|
| 138 |
last_action_error: str | None = None
|
| 139 |
-
score_breakdown: dict[str, float] = Field(default_factory=dict)
|
| 140 |
-
missing_items: list[str] = Field(default_factory=list)
|
| 141 |
submitted: bool = False
|
| 142 |
step_count: int = 0
|
| 143 |
|
|
|
|
| 80 |
ticket: SupportTicket
|
| 81 |
policy_notes: list[str] = Field(default_factory=list)
|
| 82 |
investigation_requirements: list[str] = Field(default_factory=list)
|
|
|
|
|
|
|
| 83 |
investigation_log: list["InvestigationRecord"] = Field(default_factory=list)
|
| 84 |
workspace: TriageWorkspace
|
| 85 |
available_actions: list[str] = Field(default_factory=list)
|
| 86 |
allowed_values: dict[str, list[str]] = Field(default_factory=dict)
|
|
|
|
|
|
|
|
|
|
| 87 |
last_action_error: str | None = None
|
| 88 |
steps_remaining: int = 0
|
| 89 |
|
|
|
|
| 97 |
investigation_log: list["InvestigationRecord"] = Field(default_factory=list)
|
| 98 |
last_action_error: str | None = None
|
| 99 |
completion_score: float = 0.0
|
|
|
|
| 100 |
submitted: bool = False
|
| 101 |
|
| 102 |
|
|
|
|
| 130 |
|
| 131 |
class TriageStepInfo(BaseModel):
|
| 132 |
last_action_error: str | None = None
|
|
|
|
|
|
|
| 133 |
submitted: bool = False
|
| 134 |
step_count: int = 0
|
| 135 |
|
tests/test_environment.py
CHANGED
|
@@ -12,6 +12,10 @@ def test_reset_selects_requested_task():
|
|
| 12 |
assert observation.workspace.priority is None
|
| 13 |
assert observation.steps_remaining == 14
|
| 14 |
assert "lookup_order(value)" in observation.available_actions
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15 |
|
| 16 |
|
| 17 |
def test_seed_selects_alternate_variant():
|
|
@@ -56,7 +60,7 @@ def test_perfect_billing_run_reaches_full_score():
|
|
| 56 |
final_observation = env.step(TriageAction(action_type="submit"))
|
| 57 |
|
| 58 |
assert final_observation.done is True
|
| 59 |
-
assert
|
| 60 |
assert env.state.submitted is True
|
| 61 |
|
| 62 |
|
|
|
|
| 12 |
assert observation.workspace.priority is None
|
| 13 |
assert observation.steps_remaining == 14
|
| 14 |
assert "lookup_order(value)" in observation.available_actions
|
| 15 |
+
assert hasattr(observation, "investigation_requirements")
|
| 16 |
+
assert not hasattr(observation, "reply_requirements")
|
| 17 |
+
assert not hasattr(observation, "score_breakdown")
|
| 18 |
+
assert not hasattr(observation, "missing_items")
|
| 19 |
|
| 20 |
|
| 21 |
def test_seed_selects_alternate_variant():
|
|
|
|
| 60 |
final_observation = env.step(TriageAction(action_type="submit"))
|
| 61 |
|
| 62 |
assert final_observation.done is True
|
| 63 |
+
assert env.state.completion_score == 1.0
|
| 64 |
assert env.state.submitted is True
|
| 65 |
|
| 66 |
|