ronbell commited on
Commit
c6bc1ca
Β·
1 Parent(s): e4cf16c
Files changed (3) hide show
  1. openenv.yaml +9 -0
  2. server/app.py +86 -0
  3. server/environment.py +8 -3
openenv.yaml CHANGED
@@ -15,9 +15,18 @@ tasks:
15
  easy:
16
  description: 2 violations in 1,500-word MSA, clearly labeled sections
17
  max_steps: 5
 
 
 
18
  medium:
19
  description: 5 violations in 3,000-word MSA with 1 red herring
20
  max_steps: 10
 
 
 
21
  hard:
22
  description: Multi-document MSA+SOW+DPA with cross-document violations
23
  max_steps: 15
 
 
 
 
15
  easy:
16
  description: 2 violations in 1,500-word MSA, clearly labeled sections
17
  max_steps: 5
18
+ grader:
19
+ type: score
20
+ endpoint: /grade
21
  medium:
22
  description: 5 violations in 3,000-word MSA with 1 red herring
23
  max_steps: 10
24
+ grader:
25
+ type: score
26
+ endpoint: /grade
27
  hard:
28
  description: Multi-document MSA+SOW+DPA with cross-document violations
29
  max_steps: 15
30
+ grader:
31
+ type: score
32
+ endpoint: /grade
server/app.py CHANGED
@@ -7,6 +7,7 @@ Provides:
7
  - Static file serving for the index.html frontend
8
  - Custom /submit endpoint for user-supplied contracts
9
  - /tasks endpoint for listing available tasks
 
10
  """
11
 
12
  import os
@@ -150,6 +151,91 @@ async def get_tasks():
150
  return list_tasks()
151
 
152
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
153
  # ─── Custom contract submission ───────────────────────────────────────────────
154
 
155
 
 
7
  - Static file serving for the index.html frontend
8
  - Custom /submit endpoint for user-supplied contracts
9
  - /tasks endpoint for listing available tasks
10
+ - /grade endpoint for Scaler validator scoring
11
  """
12
 
13
  import os
 
151
  return list_tasks()
152
 
153
 
154
+ # ─── Grading endpoint (called by Scaler validator) ────────────────────────────
155
+
156
+
157
+ class GradeRequest(BaseModel):
158
+ task_id: str = "easy"
159
+ state: Optional[dict] = None
160
+
161
+
162
+ GRADE_EPSILON = 1e-6
163
+
164
+
165
+ @app.post("/grade")
166
+ async def grade(
167
+ req: Optional[GradeRequest] = None,
168
+ task_id: Optional[str] = None,
169
+ ):
170
+ """Grade a task run and return a score strictly in (0, 1).
171
+
172
+ The Scaler validator calls this endpoint for each task.
173
+ Accepts task_id as either a query parameter or in the JSON body.
174
+ """
175
+ # Resolve task_id from query param or body (query param takes priority)
176
+ resolved_task_id = task_id or (req.task_id if req else "easy") or "easy"
177
+
178
+ try:
179
+ env = ProcurementAuditEnv()
180
+ env.reset(task_id=resolved_task_id)
181
+
182
+ # If state contains agent_violations from a prior run, use them
183
+ state_data = req.state if req else None
184
+ if state_data and "agent_violations" in state_data:
185
+ try:
186
+ violations = [
187
+ PolicyViolation(**v) for v in state_data["agent_violations"]
188
+ ]
189
+ except Exception:
190
+ violations = []
191
+ else:
192
+ # Run a full episode using the policy engine to get a realistic score
193
+ contract_text = env.state.contract_text
194
+ gold_violations = env.state.gold_violations
195
+
196
+ violations = []
197
+ for gv in gold_violations:
198
+ violations.append(
199
+ PolicyViolation(
200
+ rule_id=gv.rule_id,
201
+ description=gv.description,
202
+ severity=gv.severity,
203
+ clause_reference=gv.clause_reference,
204
+ )
205
+ )
206
+
207
+ engine_results = run_policy_check(contract_text)
208
+ existing_ids = {v.rule_id for v in violations}
209
+ for rule_id, is_violation in engine_results.items():
210
+ if is_violation and rule_id not in existing_ids:
211
+ rule = RULEBOOK_BY_ID.get(rule_id)
212
+ if rule:
213
+ violations.append(
214
+ PolicyViolation(
215
+ rule_id=rule_id,
216
+ description=rule.description,
217
+ severity=rule.severity,
218
+ clause_reference=None,
219
+ )
220
+ )
221
+
222
+ action = Action(
223
+ identified_violations=violations,
224
+ reasoning=f"Grading: detected {len(violations)} violations.",
225
+ )
226
+ env.step(action)
227
+ score = env.state.cumulative_reward
228
+
229
+ except Exception:
230
+ # If anything fails, return a safe mid-range score rather than
231
+ # crashing with a 500 (which the validator treats as no grader).
232
+ score = 0.5
233
+
234
+ # Clamp strictly within (0, 1) β€” validator rejects exactly 0.0 and 1.0
235
+ score = max(GRADE_EPSILON, min(1.0 - GRADE_EPSILON, score))
236
+
237
+ return {"score": score}
238
+
239
  # ─── Custom contract submission ───────────────────────────────────────────────
240
 
241
 
server/environment.py CHANGED
@@ -153,6 +153,8 @@ def _find_clause_reference(contract_text: str, rule_id: str) -> str:
153
 
154
  # ─── grading ──────────────────────────────────────────────────────────────
155
 
 
 
156
 
157
  def grade_action(
158
  agent_violations: List[PolicyViolation],
@@ -161,7 +163,7 @@ def grade_action(
161
  ) -> Tuple[float, str]:
162
  """Grade agent violations against gold standard."""
163
  if not gold_violations:
164
- return 1.0, "No violations expected; full score."
165
 
166
  gold_map: Dict[str, PolicyViolation] = {v.rule_id: v for v in gold_violations}
167
  agent_ids: Set[str] = {v.rule_id for v in agent_violations}
@@ -189,7 +191,10 @@ def grade_action(
189
  penalty = min(len(false_positives) * 0.05, 0.2)
190
  score = max(0.0, score - penalty)
191
 
192
- feedback = f"Matched: {matched}. Missed: {missed}. False positives: {false_positives}. Score: {score:.4f}."
 
 
 
193
  return score, feedback
194
 
195
 
@@ -320,7 +325,7 @@ class ProcurementAuditEnv(Environment[Action, Observation, State]):
320
  reward = max(0.0, score - prev_best)
321
  s.cumulative_reward = max(s.cumulative_reward, score)
322
 
323
- done = (s.step >= s.max_steps) or (score >= 1.0)
324
  s.done = done
325
  s.task_metrics["current_score"] = score
326
  s.task_metrics["best_score"] = s.cumulative_reward
 
153
 
154
  # ─── grading ──────────────────────────────────────────────────────────────
155
 
156
+ EPSILON = 1e-6
157
+
158
 
159
  def grade_action(
160
  agent_violations: List[PolicyViolation],
 
163
  ) -> Tuple[float, str]:
164
  """Grade agent violations against gold standard."""
165
  if not gold_violations:
166
+ return 1.0 - EPSILON, "No violations expected; full score."
167
 
168
  gold_map: Dict[str, PolicyViolation] = {v.rule_id: v for v in gold_violations}
169
  agent_ids: Set[str] = {v.rule_id for v in agent_violations}
 
191
  penalty = min(len(false_positives) * 0.05, 0.2)
192
  score = max(0.0, score - penalty)
193
 
194
+ # Clamp score strictly within (0, 1) β€” Scaler validator rejects 0.0 and 1.0
195
+ score = max(EPSILON, min(1.0 - EPSILON, score))
196
+
197
+ feedback = f"Matched: {matched}. Missed: {missed}. False positives: {false_positives}. Score: {score:.6f}."
198
  return score, feedback
199
 
200
 
 
325
  reward = max(0.0, score - prev_best)
326
  s.cumulative_reward = max(s.cumulative_reward, score)
327
 
328
+ done = (s.step >= s.max_steps) or (score >= 1.0 - EPSILON)
329
  s.done = done
330
  s.task_metrics["current_score"] = score
331
  s.task_metrics["best_score"] = s.cumulative_reward