CoBeDigger commited on
Commit
e91d5da
·
1 Parent(s): 023d346

fix(round2): fix Stage 3 string matching in baseline script

Browse files
app/__pycache__/environment.cpython-313.pyc CHANGED
Binary files a/app/__pycache__/environment.cpython-313.pyc and b/app/__pycache__/environment.cpython-313.pyc differ
 
app/__pycache__/main.cpython-313.pyc CHANGED
Binary files a/app/__pycache__/main.cpython-313.pyc and b/app/__pycache__/main.cpython-313.pyc differ
 
app/__pycache__/models.cpython-313.pyc CHANGED
Binary files a/app/__pycache__/models.cpython-313.pyc and b/app/__pycache__/models.cpython-313.pyc differ
 
app/__pycache__/reward.cpython-313.pyc CHANGED
Binary files a/app/__pycache__/reward.cpython-313.pyc and b/app/__pycache__/reward.cpython-313.pyc differ
 
app/environment.py CHANGED
@@ -1,68 +1,103 @@
1
- import sys
2
- import subprocess
3
- import tempfile
4
  import shutil
 
 
 
5
  from pathlib import Path
6
 
7
  import pandas as pd
8
 
9
- from app.models import Action, Observation, Reward, StepResponse, StateResponse
10
- from app.tasks import task1_easy, task2_medium, task3_hard
11
- from app.graders import grader1, grader2, grader3
12
- from app.reward import RewardEngine
 
 
 
 
 
 
 
13
 
14
  class DataEngEnvironment:
15
  def __init__(self):
16
- self.current_task_id: int = 0
17
  self.step_number: int = 0
 
 
 
 
 
 
 
18
  self.current_script: str = ""
19
  self.dataframe: pd.DataFrame | None = None
20
  self.task_description: str = ""
21
  self.done: bool = False
22
  self.last_run_output: str = ""
23
  self.last_run_error: str | None = None
24
-
25
- # Privacy variables for Task 3 held-out sets
26
  self._held_out_X = None
27
  self._held_out_y = None
28
  self._full_df = None
29
-
30
- self._reward_engine = RewardEngine()
 
 
31
 
32
- def reset(self, task_id: int) -> Observation:
33
- self.current_task_id = task_id
34
  self.step_number = 0
 
 
 
 
 
 
35
  self.done = False
36
  self.last_run_output = ""
37
  self.last_run_error = None
38
-
 
 
 
 
 
 
 
 
 
39
  self._held_out_X = None
40
  self._held_out_y = None
41
  self._full_df = None
42
-
43
- self._reward_engine = RewardEngine()
44
-
45
- if task_id == 1:
46
- scenario = task1_easy.generate_scenario()
47
- elif task_id == 2:
48
- scenario = task2_medium.generate_scenario()
49
- elif task_id == 3:
50
- scenario = task3_hard.generate_scenario()
51
- # Strict assignment isolating testing datasets
52
  self._held_out_X = scenario.get("held_out_X")
53
  self._held_out_y = scenario.get("held_out_y")
54
  self._full_df = scenario.get("full_df")
 
 
 
55
  else:
56
- raise ValueError(f"Unknown task_id: {task_id}")
57
-
58
  self.current_script = scenario.get("broken_script", "")
59
  self.dataframe = scenario.get("dataframe")
60
  self.task_description = scenario.get("task_description", "")
61
  self.last_run_error = scenario.get("initial_error_log")
62
 
63
- return self._get_observation()
64
-
65
- def _get_observation(self, partial_schema: dict | None = None, partial_preview: str = "") -> Observation:
 
 
66
  return Observation(
67
  task_id=self.current_task_id,
68
  step_number=self.step_number,
@@ -71,152 +106,234 @@ class DataEngEnvironment:
71
  last_run_error=self.last_run_error,
72
  data_preview=partial_preview,
73
  schema_info=partial_schema or {},
74
- done=self.done
 
 
 
 
 
75
  )
76
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
77
  def step(self, action: Action) -> StepResponse:
 
 
 
 
 
 
 
78
  self.step_number += 1
79
-
 
80
  preview = ""
81
  schema = {}
82
  score = 0.0
83
-
 
84
  try:
85
  if action.action_type == "inspect_data":
86
  df = self.dataframe
87
  if df is not None:
88
- # Build data_preview
89
- data_preview = f"Shape: {df.shape}\n\n"
90
- data_preview += f"Columns: {list(df.columns)}\n\n"
91
- data_preview += f"Dtypes:\n{df.dtypes.to_string()}\n\n"
92
- data_preview += f"Null counts:\n{df.isnull().sum().to_string()}\n\n"
93
- data_preview += f"Sample (5 rows):\n{df.head().to_string()}\n\n"
94
- data_preview += f"Numeric stats:\n{df.describe().to_string()}"
95
- preview = data_preview
96
-
97
- # Build schema_info
98
- schema_info = {
99
  "columns": list(df.columns),
100
  "dtypes": {col: str(df[col].dtype) for col in df.columns},
101
  "null_counts": df.isnull().sum().to_dict(),
102
- "shape": list(df.shape)
103
  }
104
- schema = schema_info
105
-
106
  elif action.action_type == "check_schema":
107
- df = self.dataframe
108
- if df is not None:
109
- schema = df.dtypes.astype(str).to_dict()
110
-
111
  elif action.action_type == "edit_script":
112
  old_text = action.payload.get("old", "")
113
  new_text = action.payload.get("new", "")
114
- if old_text:
 
 
 
 
115
  if old_text not in self.current_script:
116
  self.last_run_error = "edit_script failed: the text to replace was not found in the script"
117
  else:
118
  self.current_script = self.current_script.replace(old_text, new_text)
119
-
 
 
120
  elif action.action_type == "run_script":
121
- # Creating sandbox subprocess per contract requirements
122
- with tempfile.TemporaryDirectory() as tmpdir:
123
- tmpdir_path = Path(tmpdir)
124
- try:
125
- if self.dataframe is not None:
126
- self.dataframe.to_csv(tmpdir_path / "data.csv", index=False)
127
-
128
- script_path = tmpdir_path / "script.py"
129
- sandbox_script = "import pandas as pd\ndf = pd.read_csv('data.csv')\n" + self.current_script
130
- script_path.write_text(sandbox_script, encoding="utf-8")
131
-
132
- result = subprocess.run(
133
- [sys.executable, str(script_path)],
134
- capture_output=True,
135
- text=True,
136
- timeout=10,
137
- cwd=tmpdir
138
- )
139
- self.last_run_output = result.stdout
140
- self.last_run_error = result.stderr if result.returncode != 0 else None
141
- except subprocess.TimeoutExpired as e:
142
- self.last_run_error = f"TimeoutExpired: script ran longer than 10s."
143
- self.last_run_output = ""
144
-
145
  elif action.action_type == "submit":
146
- if self.current_task_id == 1:
147
- score = grader1.grade(self.current_script, self.last_run_output, self.last_run_error)
148
- elif self.current_task_id == 2:
149
- score = grader2.grade(self.current_script, self.last_run_output, self.last_run_error)
150
- elif self.current_task_id == 3:
151
- score = grader3.grade(self.current_script, self._full_df)
152
-
153
- self.done = True
154
-
155
- except Exception as e:
156
- self.last_run_error = f"Action Handler Error: {str(e)}"
157
-
158
- # Maximum step bound limit restriction per project contract requirements
159
- if self.step_number >= 20:
160
  self.done = True
161
-
 
162
  obs = self._get_observation(partial_schema=schema, partial_preview=preview)
163
-
164
- # Obtain reward from Reward engine
165
- try:
166
- reward = self._reward_engine.compute_reward(
167
- task_id=self.current_task_id,
168
- action_type=action.action_type,
169
- action_payload=action.payload,
170
- observation=obs.model_dump(),
171
- grader_score=score,
172
- step_number=self.step_number
173
- )
174
- except Exception as e:
175
- # Fallback if RewardEngine throws error based on parameter changes
176
- reward = Reward(
177
- score=score,
178
- partial_rewards={},
179
- message=f"Reward calculation fallback. {e}",
180
- is_terminal=self.done
181
- )
182
-
183
- return StepResponse(observation=obs, reward=reward)
184
 
185
- def submit_grader(self, task_id: int) -> Reward:
186
- score = 0.0
187
- if task_id == 1:
188
- score = grader1.grade(self.current_script, self.last_run_output, self.last_run_error)
189
- elif task_id == 2:
190
- score = grader2.grade(self.current_script, self.last_run_output, self.last_run_error)
191
- elif task_id == 3:
192
- score = grader3.grade(self.current_script, self._full_df)
193
-
194
- obs = self._get_observation()
195
-
196
- try:
197
- reward = self._reward_engine.compute_reward(
198
- task_id=task_id,
199
- action_type="submit",
200
- action_payload={},
201
- observation=obs.model_dump(),
202
- grader_score=score,
203
- step_number=self.step_number
204
- )
205
- except Exception as e:
206
- reward = Reward(
207
- score=score,
208
- partial_rewards={},
209
- message=f"Reward calculation fallback. {e}",
210
- is_terminal=True
211
- )
212
- return reward
213
 
214
  def state(self) -> StateResponse:
215
  return StateResponse(
216
  task_id=self.current_task_id,
217
  step_number=self.step_number,
218
- max_steps=20,
219
  current_script=self.current_script,
220
  episode_done=self.done,
221
- task_description=self.task_description
 
 
 
 
 
 
222
  )
 
1
+ from __future__ import annotations
2
+
 
3
  import shutil
4
+ import subprocess
5
+ import sys
6
+ import uuid
7
  from pathlib import Path
8
 
9
  import pandas as pd
10
 
11
+ from app.actors.code_reviewer import CodeReviewer
12
+ from app.actors.mlops_bot import MLOpsBot
13
+ from app.graders import grader_stage1, grader_stage2, grader_stage3, grader_stage4
14
+ from app.models import Action, Observation, Reward, StateResponse, StepResponse
15
+ from app.tasks import (
16
+ stage1_data_repair,
17
+ stage2_training_monitor,
18
+ stage3_eval_validation,
19
+ stage4_deploy_gate,
20
+ )
21
+
22
 
23
  class DataEngEnvironment:
24
  def __init__(self):
25
+ self.current_task_id: int = 1
26
  self.step_number: int = 0
27
+ self.current_stage: int = 1
28
+ self.stage_step_number: int = 0
29
+ self.loop_count: int = 0
30
+ self.stages_completed: list[int] = []
31
+ self.episode_score: float = 0.0
32
+ self.actor_feedback: str = ""
33
+
34
  self.current_script: str = ""
35
  self.dataframe: pd.DataFrame | None = None
36
  self.task_description: str = ""
37
  self.done: bool = False
38
  self.last_run_output: str = ""
39
  self.last_run_error: str | None = None
40
+
 
41
  self._held_out_X = None
42
  self._held_out_y = None
43
  self._full_df = None
44
+ self._deploy_conditions = None
45
+
46
+ self._code_reviewer = CodeReviewer()
47
+ self._mlops_bot = MLOpsBot()
48
 
49
+ def reset(self, task_id: int | None = None) -> Observation:
50
+ self.current_task_id = 1
51
  self.step_number = 0
52
+ self.current_stage = 1
53
+ self.stage_step_number = 0
54
+ self.loop_count = 0
55
+ self.stages_completed = []
56
+ self.episode_score = 0.0
57
+ self.actor_feedback = ""
58
  self.done = False
59
  self.last_run_output = ""
60
  self.last_run_error = None
61
+ self._load_stage(1)
62
+ return self._get_observation()
63
+
64
+ def _load_stage(self, stage: int) -> None:
65
+ self.current_stage = stage
66
+ self.current_task_id = stage
67
+ self.stage_step_number = 0
68
+ self.actor_feedback = ""
69
+ self.last_run_output = ""
70
+
71
  self._held_out_X = None
72
  self._held_out_y = None
73
  self._full_df = None
74
+ self._deploy_conditions = None
75
+
76
+ if stage == 1:
77
+ scenario = stage1_data_repair.generate_scenario()
78
+ elif stage == 2:
79
+ scenario = stage2_training_monitor.generate_scenario()
80
+ elif stage == 3:
81
+ scenario = stage3_eval_validation.generate_scenario()
 
 
82
  self._held_out_X = scenario.get("held_out_X")
83
  self._held_out_y = scenario.get("held_out_y")
84
  self._full_df = scenario.get("full_df")
85
+ elif stage == 4:
86
+ scenario = stage4_deploy_gate.generate_scenario()
87
+ self._deploy_conditions = scenario.get("deploy_conditions")
88
  else:
89
+ raise ValueError(f"Unknown stage: {stage}")
90
+
91
  self.current_script = scenario.get("broken_script", "")
92
  self.dataframe = scenario.get("dataframe")
93
  self.task_description = scenario.get("task_description", "")
94
  self.last_run_error = scenario.get("initial_error_log")
95
 
96
+ def _get_observation(
97
+ self,
98
+ partial_schema: dict | None = None,
99
+ partial_preview: str = "",
100
+ ) -> Observation:
101
  return Observation(
102
  task_id=self.current_task_id,
103
  step_number=self.step_number,
 
106
  last_run_error=self.last_run_error,
107
  data_preview=partial_preview,
108
  schema_info=partial_schema or {},
109
+ done=self.done,
110
+ current_stage=self.current_stage,
111
+ stage_step_number=self.stage_step_number,
112
+ actor_feedback=self.actor_feedback,
113
+ loop_count=self.loop_count,
114
+ stages_completed=list(self.stages_completed),
115
  )
116
 
117
+ def _make_reward(self, score: float, message: str, is_terminal: bool) -> Reward:
118
+ return Reward(
119
+ score=float(max(0.0, min(1.0, score))),
120
+ partial_rewards={
121
+ "episode_score": float(max(0.0, min(1.0, self.episode_score))),
122
+ "current_stage": float(self.current_stage),
123
+ },
124
+ message=message,
125
+ is_terminal=is_terminal,
126
+ )
127
+
128
+ def _run_current_script(self) -> None:
129
+ runs_dir = Path.cwd() / ".env_runs"
130
+ runs_dir.mkdir(exist_ok=True)
131
+ tmpdir = runs_dir / f"stage_{self.current_stage}_{uuid.uuid4().hex}"
132
+ tmpdir.mkdir()
133
+
134
+ def cleanup() -> None:
135
+ shutil.rmtree(tmpdir, ignore_errors=True)
136
+
137
+ try:
138
+ if self.dataframe is not None:
139
+ self.dataframe.to_csv(tmpdir / "data.csv", index=False)
140
+
141
+ script_path = tmpdir / "script.py"
142
+ sandbox_script = (
143
+ "import pandas as pd\n"
144
+ "df = pd.read_csv('data.csv')\n"
145
+ + self.current_script
146
+ )
147
+ script_path.write_text(sandbox_script, encoding="utf-8")
148
+
149
+ result = subprocess.run(
150
+ [sys.executable, str(script_path)],
151
+ capture_output=True,
152
+ text=True,
153
+ timeout=10,
154
+ cwd=tmpdir,
155
+ check=False,
156
+ )
157
+ self.last_run_output = result.stdout
158
+ self.last_run_error = result.stderr if result.returncode != 0 else None
159
+ except subprocess.TimeoutExpired:
160
+ self.last_run_error = "TimeoutExpired: script ran longer than 10s."
161
+ self.last_run_output = ""
162
+ finally:
163
+ cleanup()
164
+
165
+ def _grade_current_stage(self) -> float:
166
+ if self.current_stage == 1:
167
+ return grader_stage1.grade(
168
+ self.current_script,
169
+ self.last_run_output,
170
+ self.last_run_error,
171
+ )
172
+ if self.current_stage == 2:
173
+ return grader_stage2.grade(
174
+ self.current_script,
175
+ self.last_run_output,
176
+ self.last_run_error,
177
+ )
178
+ if self.current_stage == 3:
179
+ return grader_stage3.grade(
180
+ self.current_script,
181
+ self._full_df,
182
+ self._held_out_X,
183
+ self._held_out_y,
184
+ )
185
+ if self.current_stage == 4:
186
+ return grader_stage4.grade(
187
+ self.current_script,
188
+ self.dataframe,
189
+ self._deploy_conditions,
190
+ )
191
+ return 0.0
192
+
193
+ def _check_stage_transition(self, score: float) -> str:
194
+ stage = self.current_stage
195
+
196
+ if score >= 0.7:
197
+ if stage not in self.stages_completed:
198
+ self.stages_completed.append(stage)
199
+ self.episode_score = max(0.0, min(1.0, self.episode_score + score * 0.25))
200
+
201
+ if stage == 4:
202
+ self.done = True
203
+ return "Stage 4 complete. Deployment approved."
204
+
205
+ self._load_stage(stage + 1)
206
+ return f"Stage {stage} complete. Advanced to Stage {self.current_stage}."
207
+
208
+ if stage == 4:
209
+ self.loop_count += 1
210
+ self.episode_score = max(0.0, self.episode_score - 0.3)
211
+ if self.loop_count >= 2:
212
+ self.done = True
213
+ return "Stage 4 failed twice. Episode terminated."
214
+
215
+ self._load_stage(1)
216
+ return "Stage 4 failed. Looping back to Stage 1."
217
+
218
+ return f"Stage {stage} score below threshold. Continue fixing this stage."
219
+
220
+ def _handle_query_actor(self) -> None:
221
+ if self.current_stage == 1:
222
+ self.actor_feedback = "No reviewer assigned to this stage."
223
+ elif self.current_stage in {2, 3}:
224
+ self.actor_feedback = self._code_reviewer.review(
225
+ self.current_script,
226
+ stage=self.current_stage,
227
+ )
228
+ elif self.current_stage == 4:
229
+ self.actor_feedback = self._mlops_bot.check_deployment(
230
+ self.current_script,
231
+ self.dataframe,
232
+ self._deploy_conditions,
233
+ )
234
+
235
  def step(self, action: Action) -> StepResponse:
236
+ if self.done:
237
+ obs = self._get_observation()
238
+ return StepResponse(
239
+ observation=obs,
240
+ reward=self._make_reward(self.episode_score, "episode already complete", True),
241
+ )
242
+
243
  self.step_number += 1
244
+ self.stage_step_number += 1
245
+
246
  preview = ""
247
  schema = {}
248
  score = 0.0
249
+ message = "no reward"
250
+
251
  try:
252
  if action.action_type == "inspect_data":
253
  df = self.dataframe
254
  if df is not None:
255
+ preview = (
256
+ f"Shape: {df.shape}\n\n"
257
+ f"Columns: {list(df.columns)}\n\n"
258
+ f"Dtypes:\n{df.dtypes.to_string()}\n\n"
259
+ f"Null counts:\n{df.isnull().sum().to_string()}\n\n"
260
+ f"Sample (5 rows):\n{df.head().to_string()}\n\n"
261
+ f"Numeric stats:\n{df.describe().to_string()}"
262
+ )
263
+ schema = {
 
 
264
  "columns": list(df.columns),
265
  "dtypes": {col: str(df[col].dtype) for col in df.columns},
266
  "null_counts": df.isnull().sum().to_dict(),
267
+ "shape": list(df.shape),
268
  }
269
+ message = "data inspected"
270
+
271
  elif action.action_type == "check_schema":
272
+ if self.dataframe is not None:
273
+ schema = self.dataframe.dtypes.astype(str).to_dict()
274
+ message = "schema checked"
275
+
276
  elif action.action_type == "edit_script":
277
  old_text = action.payload.get("old", "")
278
  new_text = action.payload.get("new", "")
279
+ replacement_script = action.payload.get("script")
280
+ if isinstance(replacement_script, str):
281
+ self.current_script = replacement_script
282
+ self.last_run_error = None
283
+ elif old_text:
284
  if old_text not in self.current_script:
285
  self.last_run_error = "edit_script failed: the text to replace was not found in the script"
286
  else:
287
  self.current_script = self.current_script.replace(old_text, new_text)
288
+ self.last_run_error = None
289
+ message = "script edited"
290
+
291
  elif action.action_type == "run_script":
292
+ self._run_current_script()
293
+ message = "script ran" if not self.last_run_error else "script failed"
294
+
295
+ elif action.action_type == "query_actor":
296
+ self._handle_query_actor()
297
+ message = "actor queried"
298
+
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
299
  elif action.action_type == "submit":
300
+ score = self._grade_current_stage()
301
+ message = self._check_stage_transition(score)
302
+
303
+ except Exception as exc:
304
+ self.last_run_error = f"Action Handler Error: {exc}"
305
+ message = self.last_run_error
306
+
307
+ if self.step_number >= 60 or self.stage_step_number >= 15:
 
 
 
 
 
 
308
  self.done = True
309
+ message = f"{message} Stage or episode step limit reached."
310
+
311
  obs = self._get_observation(partial_schema=schema, partial_preview=preview)
312
+ reward_score = self.episode_score if action.action_type == "submit" else 0.0
313
+ if action.action_type == "submit" and not self.done:
314
+ reward_score = score
315
+ return StepResponse(
316
+ observation=obs,
317
+ reward=self._make_reward(reward_score, message, self.done),
318
+ )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
319
 
320
+ def submit_grader(self, task_id: int | None = None) -> Reward:
321
+ score = self._grade_current_stage()
322
+ message = self._check_stage_transition(score)
323
+ return self._make_reward(self.episode_score if self.done else score, message, self.done)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
324
 
325
  def state(self) -> StateResponse:
326
  return StateResponse(
327
  task_id=self.current_task_id,
328
  step_number=self.step_number,
329
+ max_steps=60,
330
  current_script=self.current_script,
331
  episode_done=self.done,
332
+ task_description=self.task_description,
333
+ current_stage=self.current_stage,
334
+ stage_step_number=self.stage_step_number,
335
+ loop_count=self.loop_count,
336
+ stages_completed=list(self.stages_completed),
337
+ episode_score=float(max(0.0, min(1.0, self.episode_score))),
338
+ actor_feedback=self.actor_feedback,
339
  )
app/main.py CHANGED
@@ -5,7 +5,12 @@ from pydantic import BaseModel
5
 
6
  from app.environment import DataEngEnvironment
7
  from app.models import Action, Observation, Reward, StepResponse, StateResponse, TaskInfo
8
- from app.tasks import task1_easy, task2_medium, task3_hard
 
 
 
 
 
9
 
10
  app = FastAPI(title="DataEngEnv API", version="1.0.0")
11
 
@@ -35,16 +40,25 @@ class BaselineResponse(BaseModel):
35
  success: bool
36
  output: str
37
 
 
 
 
 
 
 
 
 
38
  @app.on_event("startup")
39
  async def startup_event():
40
  print("DataEngEnv ready")
41
- # Call generate_scenario() on all 3 tasks to validate they load properly at startup
42
  try:
43
- task1_easy.generate_scenario()
44
- task2_medium.generate_scenario()
45
- task3_hard.generate_scenario()
 
46
  except Exception as e:
47
- print(f"Warning: A task failed to load during startup validation: {e}")
48
 
49
 
50
  @app.post("/reset", response_model=Observation | ErrorResponse)
@@ -72,26 +86,39 @@ async def api_state():
72
  @app.get("/tasks", response_model=list[TaskInfo] | ErrorResponse)
73
  async def api_tasks():
74
  try:
 
 
 
 
 
 
75
  return [
76
  TaskInfo(
77
  task_id=1,
78
- name="Easy Data Cleaning",
79
  difficulty="easy",
80
- description="Fix the broken script by identifying and replacing the incorrect column name ('age_years' to 'age').",
81
  action_schema={}
82
  ),
83
  TaskInfo(
84
  task_id=2,
85
- name="Medium Data Imputation",
86
  difficulty="medium",
87
- description="Detect missing values and outliers in the numeric dataset. Apply dropna/fillna for NaNs and clip/quantile for outliers.",
88
  action_schema={}
89
  ),
90
  TaskInfo(
91
  task_id=3,
92
- name="Hard Data Leakage",
 
 
 
 
 
 
 
93
  difficulty="hard",
94
- description="Fix the critical logical data leakage issue by moving the scaler.fit() call AFTER train_test_split() to avoid cheating on the validation set.",
95
  action_schema={}
96
  )
97
  ]
@@ -103,11 +130,25 @@ async def api_grader(req: TaskIdRequest | None = None):
103
  try:
104
  tid = req.task_id if req is not None and req.task_id is not None else env.current_task_id
105
  if tid != env.current_task_id:
106
- return ErrorResponse(error="task_id mismatch — call /reset with this task_id first")
107
  return env.submit_grader(tid)
108
  except Exception as e:
109
  return ErrorResponse(error=str(e))
110
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
111
  @app.get("/baseline", response_model=BaselineResponse | ErrorResponse)
112
  async def api_baseline():
113
  try:
 
5
 
6
  from app.environment import DataEngEnvironment
7
  from app.models import Action, Observation, Reward, StepResponse, StateResponse, TaskInfo
8
+ from app.tasks import (
9
+ stage1_data_repair,
10
+ stage2_training_monitor,
11
+ stage3_eval_validation,
12
+ stage4_deploy_gate,
13
+ )
14
 
15
  app = FastAPI(title="DataEngEnv API", version="1.0.0")
16
 
 
40
  success: bool
41
  output: str
42
 
43
+ class PipelineStatusResponse(BaseModel):
44
+ current_stage: int
45
+ loop_count: int
46
+ stages_completed: list[int]
47
+ total_steps: int
48
+ episode_score: float
49
+ actor_feedback: str
50
+
51
  @app.on_event("startup")
52
  async def startup_event():
53
  print("DataEngEnv ready")
54
+ # Call generate_scenario() on all 4 stages to validate they load properly at startup
55
  try:
56
+ stage1_data_repair.generate_scenario()
57
+ stage2_training_monitor.generate_scenario()
58
+ stage3_eval_validation.generate_scenario()
59
+ stage4_deploy_gate.generate_scenario()
60
  except Exception as e:
61
+ print(f"Warning: A stage failed to load during startup validation: {e}")
62
 
63
 
64
  @app.post("/reset", response_model=Observation | ErrorResponse)
 
86
  @app.get("/tasks", response_model=list[TaskInfo] | ErrorResponse)
87
  async def api_tasks():
88
  try:
89
+ scenarios = {
90
+ 1: stage1_data_repair.generate_scenario(),
91
+ 2: stage2_training_monitor.generate_scenario(),
92
+ 3: stage3_eval_validation.generate_scenario(),
93
+ 4: stage4_deploy_gate.generate_scenario(),
94
+ }
95
  return [
96
  TaskInfo(
97
  task_id=1,
98
+ name="Stage 1: Data Repair",
99
  difficulty="easy",
100
+ description=scenarios[1]["task_description"],
101
  action_schema={}
102
  ),
103
  TaskInfo(
104
  task_id=2,
105
+ name="Stage 2: Training Monitor",
106
  difficulty="medium",
107
+ description=scenarios[2]["task_description"],
108
  action_schema={}
109
  ),
110
  TaskInfo(
111
  task_id=3,
112
+ name="Stage 3: Eval Validation",
113
+ difficulty="hard",
114
+ description=scenarios[3]["task_description"],
115
+ action_schema={}
116
+ ),
117
+ TaskInfo(
118
+ task_id=4,
119
+ name="Stage 4: Deployment Gate",
120
  difficulty="hard",
121
+ description=scenarios[4]["task_description"],
122
  action_schema={}
123
  )
124
  ]
 
130
  try:
131
  tid = req.task_id if req is not None and req.task_id is not None else env.current_task_id
132
  if tid != env.current_task_id:
133
+ return ErrorResponse(error="task_id mismatch - call /reset first; the pipeline controls stage advancement")
134
  return env.submit_grader(tid)
135
  except Exception as e:
136
  return ErrorResponse(error=str(e))
137
 
138
+ @app.get("/pipeline_status", response_model=PipelineStatusResponse | ErrorResponse)
139
+ async def api_pipeline_status():
140
+ try:
141
+ return PipelineStatusResponse(
142
+ current_stage=env.current_stage,
143
+ loop_count=env.loop_count,
144
+ stages_completed=list(env.stages_completed),
145
+ total_steps=env.step_number,
146
+ episode_score=float(env.episode_score),
147
+ actor_feedback=env.actor_feedback,
148
+ )
149
+ except Exception as e:
150
+ return ErrorResponse(error=str(e))
151
+
152
  @app.get("/baseline", response_model=BaselineResponse | ErrorResponse)
153
  async def api_baseline():
154
  try:
app/models.py CHANGED
@@ -1,14 +1,13 @@
1
  from pydantic import BaseModel, Field, field_validator
2
- import typing
3
 
4
  class Action(BaseModel):
5
- action_type: str = Field(description="One of: inspect_data | run_script | edit_script | check_schema | submit")
6
  payload: dict = Field(description="Action payload details like patch content etc.")
7
 
8
  @field_validator("action_type")
9
  @classmethod
10
  def validate_action_type(cls, v: str) -> str:
11
- allowed = {"inspect_data", "run_script", "edit_script", "check_schema", "submit"}
12
  if v not in allowed:
13
  raise ValueError(f"action_type must be one of {allowed}")
14
  return v
@@ -22,6 +21,11 @@ class Observation(BaseModel):
22
  data_preview: str = Field(description="Preview of the current Pandas dataframe")
23
  schema_info: dict = Field(description="Column types and info")
24
  done: bool = Field(description="Whether the environment episode is complete")
 
 
 
 
 
25
 
26
  class Reward(BaseModel):
27
  score: float = Field(description="Reward score from 0.0 to 1.0")
@@ -32,8 +36,8 @@ class Reward(BaseModel):
32
  @field_validator("score")
33
  @classmethod
34
  def validate_score(cls, v: float) -> float:
35
- if not (0.0 < v < 1.0):
36
- raise ValueError("Score must be strictly between 0.0 and 1.0 (exclusive)")
37
  return v
38
 
39
  class StepResponse(BaseModel):
@@ -47,6 +51,12 @@ class StateResponse(BaseModel):
47
  current_script: str = Field(description="Content of the active python script")
48
  episode_done: bool = Field(description="Whether the episode is done")
49
  task_description: str = Field(description="Task objective description")
 
 
 
 
 
 
50
 
51
  class TaskInfo(BaseModel):
52
  task_id: int = Field(description="ID of the task")
 
1
  from pydantic import BaseModel, Field, field_validator
 
2
 
3
  class Action(BaseModel):
4
+ action_type: str = Field(description="One of: inspect_data | run_script | edit_script | check_schema | query_actor | submit")
5
  payload: dict = Field(description="Action payload details like patch content etc.")
6
 
7
  @field_validator("action_type")
8
  @classmethod
9
  def validate_action_type(cls, v: str) -> str:
10
+ allowed = {"inspect_data", "run_script", "edit_script", "check_schema", "query_actor", "submit"}
11
  if v not in allowed:
12
  raise ValueError(f"action_type must be one of {allowed}")
13
  return v
 
21
  data_preview: str = Field(description="Preview of the current Pandas dataframe")
22
  schema_info: dict = Field(description="Column types and info")
23
  done: bool = Field(description="Whether the environment episode is complete")
24
+ current_stage: int = Field(default=1, description="Current pipeline stage")
25
+ stage_step_number: int = Field(default=0, description="Step count within current stage")
26
+ actor_feedback: str = Field(default="", description="Most recent actor bot feedback")
27
+ loop_count: int = Field(default=0, description="Number of deployment failure loops")
28
+ stages_completed: list[int] = Field(default_factory=list, description="Completed pipeline stages")
29
 
30
  class Reward(BaseModel):
31
  score: float = Field(description="Reward score from 0.0 to 1.0")
 
36
  @field_validator("score")
37
  @classmethod
38
  def validate_score(cls, v: float) -> float:
39
+ if not (0.0 <= v <= 1.0):
40
+ raise ValueError("Score must be between 0.0 and 1.0 inclusive")
41
  return v
42
 
43
  class StepResponse(BaseModel):
 
51
  current_script: str = Field(description="Content of the active python script")
52
  episode_done: bool = Field(description="Whether the episode is done")
53
  task_description: str = Field(description="Task objective description")
54
+ current_stage: int = Field(default=1, description="Current pipeline stage")
55
+ stage_step_number: int = Field(default=0, description="Step count within current stage")
56
+ loop_count: int = Field(default=0, description="Number of deployment failure loops")
57
+ stages_completed: list[int] = Field(default_factory=list, description="Completed pipeline stages")
58
+ episode_score: float = Field(default=0.0, description="Cumulative 4-stage episode score")
59
+ actor_feedback: str = Field(default="", description="Most recent actor bot feedback")
60
 
61
  class TaskInfo(BaseModel):
62
  task_id: int = Field(description="ID of the task")
baseline/results.json CHANGED
@@ -1,5 +1,11 @@
1
- [
2
- {"task_id": 1, "name": "column_rename_bug", "difficulty": "easy", "score": 1.0, "steps": 3},
3
- {"task_id": 2, "name": "dirty_data", "difficulty": "medium", "score": 1.0, "steps": 15},
4
- {"task_id": 3, "name": "data_leakage", "difficulty": "hard", "score": 1.0, "steps": 15}
5
- ]
 
 
 
 
 
 
 
1
+ {
2
+ "final_score": 1.0,
3
+ "total_steps": 18,
4
+ "stages_completed": [
5
+ 1,
6
+ 2,
7
+ 3,
8
+ 4
9
+ ],
10
+ "episode_score": 1.0
11
+ }
baseline/run_baseline.py CHANGED
@@ -91,8 +91,8 @@ def run_episode():
91
  action = {"action_type": "run_script", "payload": {}}
92
  elif current_stage == 3 and stage_step == 1:
93
  action = {"action_type": "edit_script", "payload": {
94
- "old": "scaler = StandardScaler()\nX_scaled = scaler.fit_transform(X) # data leakage: fit on full data\nX_train, X_test, y_train, y_test = train_test_split(X_scaled, y, test_size=0.2, random_state=42)",
95
- "new": "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\nscaler = StandardScaler()\nX_train = scaler.fit_transform(X_train)\nX_test = scaler.transform(X_test)"
96
  }}
97
  elif current_stage == 3 and stage_step == 2:
98
  action = {"action_type": "run_script", "payload": {}}
 
91
  action = {"action_type": "run_script", "payload": {}}
92
  elif current_stage == 3 and stage_step == 1:
93
  action = {"action_type": "edit_script", "payload": {
94
+ "old": "scaler = StandardScaler()\nX_scaled = scaler.fit_transform(X)\nX_train, X_test, y_train, y_test = train_test_split(\n X_scaled, y, test_size=0.2, random_state=42, stratify=y\n)",
95
+ "new": "X_train, X_test, y_train, y_test = train_test_split(\n X, y, test_size=0.2, random_state=42, stratify=y\n)\nscaler = StandardScaler()\nX_train = scaler.fit_transform(X_train)\nX_test = scaler.transform(X_test)"
96
  }}
97
  elif current_stage == 3 and stage_step == 2:
98
  action = {"action_type": "run_script", "payload": {}}
pyproject.toml CHANGED
@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
6
  name = "dataengenv"
7
  version = "1.0.0"
8
  description = "ML Data Pipeline Debugger - OpenEnv Environment"
9
- requires-python = ">=3.10, <3.13"
10
  dependencies = [
11
  "fastapi==0.115.6",
12
  "uvicorn",
 
6
  name = "dataengenv"
7
  version = "1.0.0"
8
  description = "ML Data Pipeline Debugger - OpenEnv Environment"
9
+ requires-python = ">=3.10"
10
  dependencies = [
11
  "fastapi==0.115.6",
12
  "uvicorn",
uv.lock CHANGED
The diff for this file is too large to render. See raw diff