Neetu143 commited on
Commit
0bbd62a
·
verified ·
1 Parent(s): a140732

Upload support triage environment

Browse files
README.md CHANGED
@@ -26,6 +26,8 @@ This environment models a real operations workflow that human support leads and
26
 
27
  The key design goal is to punish shallow “form filling.” An agent can no longer get a high score just by memorizing queue and status values. It must look up order, account, policy, or fraud signals first, and the grader rewards or penalizes that investigation path directly.
28
 
 
 
29
  ## Task families
30
 
31
  The environment ships with three task families and seeded procedural variants inside each family:
@@ -82,7 +84,6 @@ Each observation includes:
82
  - the current structured workspace
83
  - lookup targets for policy, order, account, and fraud actions
84
  - valid actions and allowed values
85
- - progress breakdown and aggregate completion score
86
  - `last_action_error`
87
  - `steps_remaining`
88
 
 
26
 
27
  The key design goal is to punish shallow “form filling.” An agent can no longer get a high score just by memorizing queue and status values. It must look up order, account, policy, or fraud signals first, and the grader rewards or penalizes that investigation path directly.
28
 
29
+ The public observation intentionally omits exact grader targets such as per-dimension score breakdowns and literal reply/note requirement slots. The agent sees the task, policies, backend evidence, and workspace state, then has to infer the correct action from that evidence.
30
+
31
  ## Task families
32
 
33
  The environment ships with three task families and seeded procedural variants inside each family:
 
84
  - the current structured workspace
85
  - lookup targets for policy, order, account, and fraud actions
86
  - valid actions and allowed values
 
87
  - `last_action_error`
88
  - `steps_remaining`
89
 
inference.py CHANGED
@@ -97,9 +97,8 @@ def _normalize_tag(value: str | None) -> str:
97
  return normalized
98
 
99
 
100
- def _covers_requirements(value: str | None, requirements: list[str]) -> bool:
101
- normalized = _normalize_text(value)
102
- return all(_normalize_text(requirement) in normalized for requirement in requirements)
103
 
104
 
105
  def _action_for_log(action: TriageAction) -> str:
@@ -133,20 +132,15 @@ def _build_prompt_with_history(observation, recent_steps: list[dict[str, str]])
133
  "ticket": observation.ticket.model_dump(),
134
  "policy_notes": observation.policy_notes,
135
  "investigation_requirements": observation.investigation_requirements,
136
- "note_requirements": observation.note_requirements,
137
- "reply_requirements": observation.reply_requirements,
138
  "investigation_log": [record.model_dump() for record in observation.investigation_log],
139
  "workspace": observation.workspace.model_dump(),
140
  "allowed_values": observation.allowed_values,
141
  "available_actions": observation.available_actions,
142
- "completion_score": observation.completion_score,
143
- "missing_items": observation.missing_items,
144
  "last_action_error": observation.last_action_error,
145
  "steps_remaining": observation.steps_remaining,
146
  "recent_steps": recent_steps[-5:],
147
  }
148
 
149
- solved_items = [key for key, score in observation.score_breakdown.items() if score >= 0.999]
150
  avoid_pairs = [
151
  step["action"]
152
  for step in recent_steps[-5:]
@@ -157,9 +151,8 @@ def _build_prompt_with_history(observation, recent_steps: list[dict[str, str]])
157
  "Look up missing evidence before finalizing the workflow.\n"
158
  "Choose exactly one next action with the choose_action function.\n"
159
  "Set value to null only for submit.\n"
160
- f"Already solved areas: {solved_items}.\n"
161
  f"Do not repeat recently bad actions: {avoid_pairs}.\n"
162
- "Prefer unsolved investigation or missing rubric items over editing solved fields.\n"
163
  f"Observation:{json.dumps(payload, separators=(',', ':'))}"
164
  )
165
 
@@ -411,10 +404,10 @@ def _planned_action(observation) -> TriageAction:
411
  if _normalize_tag(tag) not in existing_tags:
412
  return TriageAction(action_type="add_tag", value=tag)
413
 
414
- if not _covers_requirements(workspace.internal_note, observation.note_requirements):
415
  return TriageAction(action_type="add_internal_note", value=plan["note"])
416
 
417
- if not _covers_requirements(workspace.draft_reply, observation.reply_requirements):
418
  return TriageAction(action_type="draft_reply", value=plan["reply"])
419
 
420
  return TriageAction(action_type="submit")
@@ -439,10 +432,10 @@ def _prefer_model_action(observation, candidate: TriageAction, planned: TriageAc
439
  return _normalize_tag(candidate.value) == _normalize_tag(planned.value)
440
 
441
  if candidate.action_type == "add_internal_note":
442
- return _covers_requirements(candidate.value, observation.note_requirements)
443
 
444
  if candidate.action_type == "draft_reply":
445
- return _covers_requirements(candidate.value, observation.reply_requirements)
446
 
447
  return candidate.action_type == "submit"
448
 
 
97
  return normalized
98
 
99
 
100
+ def _matches_expected_text(value: str | None, expected: str) -> bool:
101
+ return _normalize_text(value) == _normalize_text(expected)
 
102
 
103
 
104
  def _action_for_log(action: TriageAction) -> str:
 
132
  "ticket": observation.ticket.model_dump(),
133
  "policy_notes": observation.policy_notes,
134
  "investigation_requirements": observation.investigation_requirements,
 
 
135
  "investigation_log": [record.model_dump() for record in observation.investigation_log],
136
  "workspace": observation.workspace.model_dump(),
137
  "allowed_values": observation.allowed_values,
138
  "available_actions": observation.available_actions,
 
 
139
  "last_action_error": observation.last_action_error,
140
  "steps_remaining": observation.steps_remaining,
141
  "recent_steps": recent_steps[-5:],
142
  }
143
 
 
144
  avoid_pairs = [
145
  step["action"]
146
  for step in recent_steps[-5:]
 
151
  "Look up missing evidence before finalizing the workflow.\n"
152
  "Choose exactly one next action with the choose_action function.\n"
153
  "Set value to null only for submit.\n"
 
154
  f"Do not repeat recently bad actions: {avoid_pairs}.\n"
155
+ "Prefer unresolved investigation and policy-backed workflow decisions over guesswork.\n"
156
  f"Observation:{json.dumps(payload, separators=(',', ':'))}"
157
  )
158
 
 
404
  if _normalize_tag(tag) not in existing_tags:
405
  return TriageAction(action_type="add_tag", value=tag)
406
 
407
+ if not _matches_expected_text(workspace.internal_note, plan["note"]):
408
  return TriageAction(action_type="add_internal_note", value=plan["note"])
409
 
410
+ if not _matches_expected_text(workspace.draft_reply, plan["reply"]):
411
  return TriageAction(action_type="draft_reply", value=plan["reply"])
412
 
413
  return TriageAction(action_type="submit")
 
432
  return _normalize_tag(candidate.value) == _normalize_tag(planned.value)
433
 
434
  if candidate.action_type == "add_internal_note":
435
+ return _matches_expected_text(candidate.value, planned.value or "")
436
 
437
  if candidate.action_type == "draft_reply":
438
+ return _matches_expected_text(candidate.value, planned.value or "")
439
 
440
  return candidate.action_type == "submit"
441
 
support_triage_env/client.py CHANGED
@@ -30,8 +30,6 @@ class SupportTriageEnv(EnvClient[TriageAction, TriageObservation, TriageState]):
30
  ticket=SupportTicket.model_validate(obs_data.get("ticket", {})),
31
  policy_notes=obs_data.get("policy_notes", []),
32
  investigation_requirements=obs_data.get("investigation_requirements", []),
33
- note_requirements=obs_data.get("note_requirements", []),
34
- reply_requirements=obs_data.get("reply_requirements", []),
35
  investigation_log=[
36
  InvestigationRecord.model_validate(item)
37
  for item in obs_data.get("investigation_log", [])
@@ -39,9 +37,6 @@ class SupportTriageEnv(EnvClient[TriageAction, TriageObservation, TriageState]):
39
  workspace=TriageWorkspace.model_validate(obs_data.get("workspace", {})),
40
  available_actions=obs_data.get("available_actions", []),
41
  allowed_values=obs_data.get("allowed_values", {}),
42
- completion_score=obs_data.get("completion_score", 0.0),
43
- score_breakdown=obs_data.get("score_breakdown", {}),
44
- missing_items=obs_data.get("missing_items", []),
45
  last_action_error=obs_data.get("last_action_error"),
46
  steps_remaining=obs_data.get("steps_remaining", 0),
47
  reward=payload.get("reward"),
@@ -68,6 +63,5 @@ class SupportTriageEnv(EnvClient[TriageAction, TriageObservation, TriageState]):
68
  ],
69
  last_action_error=payload.get("last_action_error"),
70
  completion_score=payload.get("completion_score", 0.0),
71
- score_breakdown=payload.get("score_breakdown", {}),
72
  submitted=payload.get("submitted", False),
73
  )
 
30
  ticket=SupportTicket.model_validate(obs_data.get("ticket", {})),
31
  policy_notes=obs_data.get("policy_notes", []),
32
  investigation_requirements=obs_data.get("investigation_requirements", []),
 
 
33
  investigation_log=[
34
  InvestigationRecord.model_validate(item)
35
  for item in obs_data.get("investigation_log", [])
 
37
  workspace=TriageWorkspace.model_validate(obs_data.get("workspace", {})),
38
  available_actions=obs_data.get("available_actions", []),
39
  allowed_values=obs_data.get("allowed_values", {}),
 
 
 
40
  last_action_error=obs_data.get("last_action_error"),
41
  steps_remaining=obs_data.get("steps_remaining", 0),
42
  reward=payload.get("reward"),
 
63
  ],
64
  last_action_error=payload.get("last_action_error"),
65
  completion_score=payload.get("completion_score", 0.0),
 
66
  submitted=payload.get("submitted", False),
67
  )
support_triage_env/environment.py CHANGED
@@ -60,7 +60,6 @@ class SupportTriageCoreEnvironment:
60
  investigation_log=[],
61
  last_action_error=None,
62
  completion_score=0.0,
63
- score_breakdown={},
64
  submitted=False,
65
  )
66
  self._last_action_signature = None
@@ -147,8 +146,6 @@ class SupportTriageCoreEnvironment:
147
  done=observation.done,
148
  info=TriageStepInfo(
149
  last_action_error=self._state.last_action_error,
150
- score_breakdown=dict(self._state.score_breakdown),
151
- missing_items=list(observation.missing_items),
152
  submitted=self._state.submitted,
153
  step_count=self._state.step_count,
154
  ),
@@ -226,7 +223,6 @@ class SupportTriageCoreEnvironment:
226
 
227
  def _sync_grade(self, grade) -> None:
228
  self._state.completion_score = grade.score
229
- self._state.score_breakdown = grade.breakdown
230
 
231
  def _run_lookup(self, action_type: str, target: str | None) -> bool:
232
  source = LOOKUP_ACTIONS[action_type]
@@ -267,15 +263,10 @@ class SupportTriageCoreEnvironment:
267
  ticket=self._task.ticket,
268
  policy_notes=self._task.policy_notes,
269
  investigation_requirements=list(self._task.investigation_requirements),
270
- note_requirements=[slot.alternatives[0] for slot in self._task.note_slots],
271
- reply_requirements=[slot.alternatives[0] for slot in self._task.reply_slots],
272
  investigation_log=[record.model_copy(deep=True) for record in self._state.investigation_log],
273
  workspace=self._state.workspace.model_copy(deep=True),
274
  available_actions=AVAILABLE_ACTIONS,
275
  allowed_values=allowed_values(self._task),
276
- completion_score=current_grade.score,
277
- score_breakdown=current_grade.breakdown,
278
- missing_items=current_grade.missing_items,
279
  last_action_error=self._state.last_action_error,
280
  steps_remaining=max(0, MAX_STEPS - self._state.step_count),
281
  reward=reward,
 
60
  investigation_log=[],
61
  last_action_error=None,
62
  completion_score=0.0,
 
63
  submitted=False,
64
  )
65
  self._last_action_signature = None
 
146
  done=observation.done,
147
  info=TriageStepInfo(
148
  last_action_error=self._state.last_action_error,
 
 
149
  submitted=self._state.submitted,
150
  step_count=self._state.step_count,
151
  ),
 
223
 
224
  def _sync_grade(self, grade) -> None:
225
  self._state.completion_score = grade.score
 
226
 
227
  def _run_lookup(self, action_type: str, target: str | None) -> bool:
228
  source = LOOKUP_ACTIONS[action_type]
 
263
  ticket=self._task.ticket,
264
  policy_notes=self._task.policy_notes,
265
  investigation_requirements=list(self._task.investigation_requirements),
 
 
266
  investigation_log=[record.model_copy(deep=True) for record in self._state.investigation_log],
267
  workspace=self._state.workspace.model_copy(deep=True),
268
  available_actions=AVAILABLE_ACTIONS,
269
  allowed_values=allowed_values(self._task),
 
 
 
270
  last_action_error=self._state.last_action_error,
271
  steps_remaining=max(0, MAX_STEPS - self._state.step_count),
272
  reward=reward,
support_triage_env/models.py CHANGED
@@ -80,15 +80,10 @@ class TriageObservation(Observation):
80
  ticket: SupportTicket
81
  policy_notes: list[str] = Field(default_factory=list)
82
  investigation_requirements: list[str] = Field(default_factory=list)
83
- note_requirements: list[str] = Field(default_factory=list)
84
- reply_requirements: list[str] = Field(default_factory=list)
85
  investigation_log: list["InvestigationRecord"] = Field(default_factory=list)
86
  workspace: TriageWorkspace
87
  available_actions: list[str] = Field(default_factory=list)
88
  allowed_values: dict[str, list[str]] = Field(default_factory=dict)
89
- completion_score: float = 0.0
90
- score_breakdown: dict[str, float] = Field(default_factory=dict)
91
- missing_items: list[str] = Field(default_factory=list)
92
  last_action_error: str | None = None
93
  steps_remaining: int = 0
94
 
@@ -102,7 +97,6 @@ class TriageState(State):
102
  investigation_log: list["InvestigationRecord"] = Field(default_factory=list)
103
  last_action_error: str | None = None
104
  completion_score: float = 0.0
105
- score_breakdown: dict[str, float] = Field(default_factory=dict)
106
  submitted: bool = False
107
 
108
 
@@ -136,8 +130,6 @@ class TriageReward(BaseModel):
136
 
137
  class TriageStepInfo(BaseModel):
138
  last_action_error: str | None = None
139
- score_breakdown: dict[str, float] = Field(default_factory=dict)
140
- missing_items: list[str] = Field(default_factory=list)
141
  submitted: bool = False
142
  step_count: int = 0
143
 
 
80
  ticket: SupportTicket
81
  policy_notes: list[str] = Field(default_factory=list)
82
  investigation_requirements: list[str] = Field(default_factory=list)
 
 
83
  investigation_log: list["InvestigationRecord"] = Field(default_factory=list)
84
  workspace: TriageWorkspace
85
  available_actions: list[str] = Field(default_factory=list)
86
  allowed_values: dict[str, list[str]] = Field(default_factory=dict)
 
 
 
87
  last_action_error: str | None = None
88
  steps_remaining: int = 0
89
 
 
97
  investigation_log: list["InvestigationRecord"] = Field(default_factory=list)
98
  last_action_error: str | None = None
99
  completion_score: float = 0.0
 
100
  submitted: bool = False
101
 
102
 
 
130
 
131
  class TriageStepInfo(BaseModel):
132
  last_action_error: str | None = None
 
 
133
  submitted: bool = False
134
  step_count: int = 0
135
 
tests/test_environment.py CHANGED
@@ -12,6 +12,10 @@ def test_reset_selects_requested_task():
12
  assert observation.workspace.priority is None
13
  assert observation.steps_remaining == 14
14
  assert "lookup_order(value)" in observation.available_actions
 
 
 
 
15
 
16
 
17
  def test_seed_selects_alternate_variant():
@@ -56,7 +60,7 @@ def test_perfect_billing_run_reaches_full_score():
56
  final_observation = env.step(TriageAction(action_type="submit"))
57
 
58
  assert final_observation.done is True
59
- assert final_observation.completion_score == 1.0
60
  assert env.state.submitted is True
61
 
62
 
 
12
  assert observation.workspace.priority is None
13
  assert observation.steps_remaining == 14
14
  assert "lookup_order(value)" in observation.available_actions
15
+ assert hasattr(observation, "investigation_requirements")
16
+ assert not hasattr(observation, "reply_requirements")
17
+ assert not hasattr(observation, "score_breakdown")
18
+ assert not hasattr(observation, "missing_items")
19
 
20
 
21
  def test_seed_selects_alternate_variant():
 
60
  final_observation = env.step(TriageAction(action_type="submit"))
61
 
62
  assert final_observation.done is True
63
+ assert env.state.completion_score == 1.0
64
  assert env.state.submitted is True
65
 
66