Spaces:
Sleeping
Sleeping
Commit ·
e91d5da
1
Parent(s): 023d346
fix(round2): fix Stage 3 string matching in baseline script
Browse files- app/__pycache__/environment.cpython-313.pyc +0 -0
- app/__pycache__/main.cpython-313.pyc +0 -0
- app/__pycache__/models.cpython-313.pyc +0 -0
- app/__pycache__/reward.cpython-313.pyc +0 -0
- app/environment.py +260 -143
- app/main.py +54 -13
- app/models.py +15 -5
- baseline/results.json +11 -5
- baseline/run_baseline.py +2 -2
- pyproject.toml +1 -1
- uv.lock +0 -0
app/__pycache__/environment.cpython-313.pyc
CHANGED
|
Binary files a/app/__pycache__/environment.cpython-313.pyc and b/app/__pycache__/environment.cpython-313.pyc differ
|
|
|
app/__pycache__/main.cpython-313.pyc
CHANGED
|
Binary files a/app/__pycache__/main.cpython-313.pyc and b/app/__pycache__/main.cpython-313.pyc differ
|
|
|
app/__pycache__/models.cpython-313.pyc
CHANGED
|
Binary files a/app/__pycache__/models.cpython-313.pyc and b/app/__pycache__/models.cpython-313.pyc differ
|
|
|
app/__pycache__/reward.cpython-313.pyc
CHANGED
|
Binary files a/app/__pycache__/reward.cpython-313.pyc and b/app/__pycache__/reward.cpython-313.pyc differ
|
|
|
app/environment.py
CHANGED
|
@@ -1,68 +1,103 @@
|
|
| 1 |
-
import
|
| 2 |
-
|
| 3 |
-
import tempfile
|
| 4 |
import shutil
|
|
|
|
|
|
|
|
|
|
| 5 |
from pathlib import Path
|
| 6 |
|
| 7 |
import pandas as pd
|
| 8 |
|
| 9 |
-
from app.
|
| 10 |
-
from app.
|
| 11 |
-
from app.graders import
|
| 12 |
-
from app.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
|
| 14 |
class DataEngEnvironment:
|
| 15 |
def __init__(self):
|
| 16 |
-
self.current_task_id: int =
|
| 17 |
self.step_number: int = 0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18 |
self.current_script: str = ""
|
| 19 |
self.dataframe: pd.DataFrame | None = None
|
| 20 |
self.task_description: str = ""
|
| 21 |
self.done: bool = False
|
| 22 |
self.last_run_output: str = ""
|
| 23 |
self.last_run_error: str | None = None
|
| 24 |
-
|
| 25 |
-
# Privacy variables for Task 3 held-out sets
|
| 26 |
self._held_out_X = None
|
| 27 |
self._held_out_y = None
|
| 28 |
self._full_df = None
|
| 29 |
-
|
| 30 |
-
|
|
|
|
|
|
|
| 31 |
|
| 32 |
-
def reset(self, task_id: int) -> Observation:
|
| 33 |
-
self.current_task_id =
|
| 34 |
self.step_number = 0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
self.done = False
|
| 36 |
self.last_run_output = ""
|
| 37 |
self.last_run_error = None
|
| 38 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 39 |
self._held_out_X = None
|
| 40 |
self._held_out_y = None
|
| 41 |
self._full_df = None
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
scenario = task3_hard.generate_scenario()
|
| 51 |
-
# Strict assignment isolating testing datasets
|
| 52 |
self._held_out_X = scenario.get("held_out_X")
|
| 53 |
self._held_out_y = scenario.get("held_out_y")
|
| 54 |
self._full_df = scenario.get("full_df")
|
|
|
|
|
|
|
|
|
|
| 55 |
else:
|
| 56 |
-
raise ValueError(f"Unknown
|
| 57 |
-
|
| 58 |
self.current_script = scenario.get("broken_script", "")
|
| 59 |
self.dataframe = scenario.get("dataframe")
|
| 60 |
self.task_description = scenario.get("task_description", "")
|
| 61 |
self.last_run_error = scenario.get("initial_error_log")
|
| 62 |
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
|
|
|
|
|
|
| 66 |
return Observation(
|
| 67 |
task_id=self.current_task_id,
|
| 68 |
step_number=self.step_number,
|
|
@@ -71,152 +106,234 @@ class DataEngEnvironment:
|
|
| 71 |
last_run_error=self.last_run_error,
|
| 72 |
data_preview=partial_preview,
|
| 73 |
schema_info=partial_schema or {},
|
| 74 |
-
done=self.done
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 75 |
)
|
| 76 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 77 |
def step(self, action: Action) -> StepResponse:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 78 |
self.step_number += 1
|
| 79 |
-
|
|
|
|
| 80 |
preview = ""
|
| 81 |
schema = {}
|
| 82 |
score = 0.0
|
| 83 |
-
|
|
|
|
| 84 |
try:
|
| 85 |
if action.action_type == "inspect_data":
|
| 86 |
df = self.dataframe
|
| 87 |
if df is not None:
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
# Build schema_info
|
| 98 |
-
schema_info = {
|
| 99 |
"columns": list(df.columns),
|
| 100 |
"dtypes": {col: str(df[col].dtype) for col in df.columns},
|
| 101 |
"null_counts": df.isnull().sum().to_dict(),
|
| 102 |
-
"shape": list(df.shape)
|
| 103 |
}
|
| 104 |
-
|
| 105 |
-
|
| 106 |
elif action.action_type == "check_schema":
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
|
| 110 |
-
|
| 111 |
elif action.action_type == "edit_script":
|
| 112 |
old_text = action.payload.get("old", "")
|
| 113 |
new_text = action.payload.get("new", "")
|
| 114 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 115 |
if old_text not in self.current_script:
|
| 116 |
self.last_run_error = "edit_script failed: the text to replace was not found in the script"
|
| 117 |
else:
|
| 118 |
self.current_script = self.current_script.replace(old_text, new_text)
|
| 119 |
-
|
|
|
|
|
|
|
| 120 |
elif action.action_type == "run_script":
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
script_path = tmpdir_path / "script.py"
|
| 129 |
-
sandbox_script = "import pandas as pd\ndf = pd.read_csv('data.csv')\n" + self.current_script
|
| 130 |
-
script_path.write_text(sandbox_script, encoding="utf-8")
|
| 131 |
-
|
| 132 |
-
result = subprocess.run(
|
| 133 |
-
[sys.executable, str(script_path)],
|
| 134 |
-
capture_output=True,
|
| 135 |
-
text=True,
|
| 136 |
-
timeout=10,
|
| 137 |
-
cwd=tmpdir
|
| 138 |
-
)
|
| 139 |
-
self.last_run_output = result.stdout
|
| 140 |
-
self.last_run_error = result.stderr if result.returncode != 0 else None
|
| 141 |
-
except subprocess.TimeoutExpired as e:
|
| 142 |
-
self.last_run_error = f"TimeoutExpired: script ran longer than 10s."
|
| 143 |
-
self.last_run_output = ""
|
| 144 |
-
|
| 145 |
elif action.action_type == "submit":
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
|
| 149 |
-
|
| 150 |
-
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
except Exception as e:
|
| 156 |
-
self.last_run_error = f"Action Handler Error: {str(e)}"
|
| 157 |
-
|
| 158 |
-
# Maximum step bound limit restriction per project contract requirements
|
| 159 |
-
if self.step_number >= 20:
|
| 160 |
self.done = True
|
| 161 |
-
|
|
|
|
| 162 |
obs = self._get_observation(partial_schema=schema, partial_preview=preview)
|
| 163 |
-
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
|
| 168 |
-
|
| 169 |
-
|
| 170 |
-
observation=obs.model_dump(),
|
| 171 |
-
grader_score=score,
|
| 172 |
-
step_number=self.step_number
|
| 173 |
-
)
|
| 174 |
-
except Exception as e:
|
| 175 |
-
# Fallback if RewardEngine throws error based on parameter changes
|
| 176 |
-
reward = Reward(
|
| 177 |
-
score=score,
|
| 178 |
-
partial_rewards={},
|
| 179 |
-
message=f"Reward calculation fallback. {e}",
|
| 180 |
-
is_terminal=self.done
|
| 181 |
-
)
|
| 182 |
-
|
| 183 |
-
return StepResponse(observation=obs, reward=reward)
|
| 184 |
|
| 185 |
-
def submit_grader(self, task_id: int) -> Reward:
|
| 186 |
-
score =
|
| 187 |
-
|
| 188 |
-
|
| 189 |
-
elif task_id == 2:
|
| 190 |
-
score = grader2.grade(self.current_script, self.last_run_output, self.last_run_error)
|
| 191 |
-
elif task_id == 3:
|
| 192 |
-
score = grader3.grade(self.current_script, self._full_df)
|
| 193 |
-
|
| 194 |
-
obs = self._get_observation()
|
| 195 |
-
|
| 196 |
-
try:
|
| 197 |
-
reward = self._reward_engine.compute_reward(
|
| 198 |
-
task_id=task_id,
|
| 199 |
-
action_type="submit",
|
| 200 |
-
action_payload={},
|
| 201 |
-
observation=obs.model_dump(),
|
| 202 |
-
grader_score=score,
|
| 203 |
-
step_number=self.step_number
|
| 204 |
-
)
|
| 205 |
-
except Exception as e:
|
| 206 |
-
reward = Reward(
|
| 207 |
-
score=score,
|
| 208 |
-
partial_rewards={},
|
| 209 |
-
message=f"Reward calculation fallback. {e}",
|
| 210 |
-
is_terminal=True
|
| 211 |
-
)
|
| 212 |
-
return reward
|
| 213 |
|
| 214 |
def state(self) -> StateResponse:
|
| 215 |
return StateResponse(
|
| 216 |
task_id=self.current_task_id,
|
| 217 |
step_number=self.step_number,
|
| 218 |
-
max_steps=
|
| 219 |
current_script=self.current_script,
|
| 220 |
episode_done=self.done,
|
| 221 |
-
task_description=self.task_description
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 222 |
)
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
|
|
|
| 3 |
import shutil
|
| 4 |
+
import subprocess
|
| 5 |
+
import sys
|
| 6 |
+
import uuid
|
| 7 |
from pathlib import Path
|
| 8 |
|
| 9 |
import pandas as pd
|
| 10 |
|
| 11 |
+
from app.actors.code_reviewer import CodeReviewer
|
| 12 |
+
from app.actors.mlops_bot import MLOpsBot
|
| 13 |
+
from app.graders import grader_stage1, grader_stage2, grader_stage3, grader_stage4
|
| 14 |
+
from app.models import Action, Observation, Reward, StateResponse, StepResponse
|
| 15 |
+
from app.tasks import (
|
| 16 |
+
stage1_data_repair,
|
| 17 |
+
stage2_training_monitor,
|
| 18 |
+
stage3_eval_validation,
|
| 19 |
+
stage4_deploy_gate,
|
| 20 |
+
)
|
| 21 |
+
|
| 22 |
|
| 23 |
class DataEngEnvironment:
|
| 24 |
def __init__(self):
|
| 25 |
+
self.current_task_id: int = 1
|
| 26 |
self.step_number: int = 0
|
| 27 |
+
self.current_stage: int = 1
|
| 28 |
+
self.stage_step_number: int = 0
|
| 29 |
+
self.loop_count: int = 0
|
| 30 |
+
self.stages_completed: list[int] = []
|
| 31 |
+
self.episode_score: float = 0.0
|
| 32 |
+
self.actor_feedback: str = ""
|
| 33 |
+
|
| 34 |
self.current_script: str = ""
|
| 35 |
self.dataframe: pd.DataFrame | None = None
|
| 36 |
self.task_description: str = ""
|
| 37 |
self.done: bool = False
|
| 38 |
self.last_run_output: str = ""
|
| 39 |
self.last_run_error: str | None = None
|
| 40 |
+
|
|
|
|
| 41 |
self._held_out_X = None
|
| 42 |
self._held_out_y = None
|
| 43 |
self._full_df = None
|
| 44 |
+
self._deploy_conditions = None
|
| 45 |
+
|
| 46 |
+
self._code_reviewer = CodeReviewer()
|
| 47 |
+
self._mlops_bot = MLOpsBot()
|
| 48 |
|
| 49 |
+
def reset(self, task_id: int | None = None) -> Observation:
|
| 50 |
+
self.current_task_id = 1
|
| 51 |
self.step_number = 0
|
| 52 |
+
self.current_stage = 1
|
| 53 |
+
self.stage_step_number = 0
|
| 54 |
+
self.loop_count = 0
|
| 55 |
+
self.stages_completed = []
|
| 56 |
+
self.episode_score = 0.0
|
| 57 |
+
self.actor_feedback = ""
|
| 58 |
self.done = False
|
| 59 |
self.last_run_output = ""
|
| 60 |
self.last_run_error = None
|
| 61 |
+
self._load_stage(1)
|
| 62 |
+
return self._get_observation()
|
| 63 |
+
|
| 64 |
+
def _load_stage(self, stage: int) -> None:
|
| 65 |
+
self.current_stage = stage
|
| 66 |
+
self.current_task_id = stage
|
| 67 |
+
self.stage_step_number = 0
|
| 68 |
+
self.actor_feedback = ""
|
| 69 |
+
self.last_run_output = ""
|
| 70 |
+
|
| 71 |
self._held_out_X = None
|
| 72 |
self._held_out_y = None
|
| 73 |
self._full_df = None
|
| 74 |
+
self._deploy_conditions = None
|
| 75 |
+
|
| 76 |
+
if stage == 1:
|
| 77 |
+
scenario = stage1_data_repair.generate_scenario()
|
| 78 |
+
elif stage == 2:
|
| 79 |
+
scenario = stage2_training_monitor.generate_scenario()
|
| 80 |
+
elif stage == 3:
|
| 81 |
+
scenario = stage3_eval_validation.generate_scenario()
|
|
|
|
|
|
|
| 82 |
self._held_out_X = scenario.get("held_out_X")
|
| 83 |
self._held_out_y = scenario.get("held_out_y")
|
| 84 |
self._full_df = scenario.get("full_df")
|
| 85 |
+
elif stage == 4:
|
| 86 |
+
scenario = stage4_deploy_gate.generate_scenario()
|
| 87 |
+
self._deploy_conditions = scenario.get("deploy_conditions")
|
| 88 |
else:
|
| 89 |
+
raise ValueError(f"Unknown stage: {stage}")
|
| 90 |
+
|
| 91 |
self.current_script = scenario.get("broken_script", "")
|
| 92 |
self.dataframe = scenario.get("dataframe")
|
| 93 |
self.task_description = scenario.get("task_description", "")
|
| 94 |
self.last_run_error = scenario.get("initial_error_log")
|
| 95 |
|
| 96 |
+
def _get_observation(
|
| 97 |
+
self,
|
| 98 |
+
partial_schema: dict | None = None,
|
| 99 |
+
partial_preview: str = "",
|
| 100 |
+
) -> Observation:
|
| 101 |
return Observation(
|
| 102 |
task_id=self.current_task_id,
|
| 103 |
step_number=self.step_number,
|
|
|
|
| 106 |
last_run_error=self.last_run_error,
|
| 107 |
data_preview=partial_preview,
|
| 108 |
schema_info=partial_schema or {},
|
| 109 |
+
done=self.done,
|
| 110 |
+
current_stage=self.current_stage,
|
| 111 |
+
stage_step_number=self.stage_step_number,
|
| 112 |
+
actor_feedback=self.actor_feedback,
|
| 113 |
+
loop_count=self.loop_count,
|
| 114 |
+
stages_completed=list(self.stages_completed),
|
| 115 |
)
|
| 116 |
|
| 117 |
+
def _make_reward(self, score: float, message: str, is_terminal: bool) -> Reward:
|
| 118 |
+
return Reward(
|
| 119 |
+
score=float(max(0.0, min(1.0, score))),
|
| 120 |
+
partial_rewards={
|
| 121 |
+
"episode_score": float(max(0.0, min(1.0, self.episode_score))),
|
| 122 |
+
"current_stage": float(self.current_stage),
|
| 123 |
+
},
|
| 124 |
+
message=message,
|
| 125 |
+
is_terminal=is_terminal,
|
| 126 |
+
)
|
| 127 |
+
|
| 128 |
+
def _run_current_script(self) -> None:
|
| 129 |
+
runs_dir = Path.cwd() / ".env_runs"
|
| 130 |
+
runs_dir.mkdir(exist_ok=True)
|
| 131 |
+
tmpdir = runs_dir / f"stage_{self.current_stage}_{uuid.uuid4().hex}"
|
| 132 |
+
tmpdir.mkdir()
|
| 133 |
+
|
| 134 |
+
def cleanup() -> None:
|
| 135 |
+
shutil.rmtree(tmpdir, ignore_errors=True)
|
| 136 |
+
|
| 137 |
+
try:
|
| 138 |
+
if self.dataframe is not None:
|
| 139 |
+
self.dataframe.to_csv(tmpdir / "data.csv", index=False)
|
| 140 |
+
|
| 141 |
+
script_path = tmpdir / "script.py"
|
| 142 |
+
sandbox_script = (
|
| 143 |
+
"import pandas as pd\n"
|
| 144 |
+
"df = pd.read_csv('data.csv')\n"
|
| 145 |
+
+ self.current_script
|
| 146 |
+
)
|
| 147 |
+
script_path.write_text(sandbox_script, encoding="utf-8")
|
| 148 |
+
|
| 149 |
+
result = subprocess.run(
|
| 150 |
+
[sys.executable, str(script_path)],
|
| 151 |
+
capture_output=True,
|
| 152 |
+
text=True,
|
| 153 |
+
timeout=10,
|
| 154 |
+
cwd=tmpdir,
|
| 155 |
+
check=False,
|
| 156 |
+
)
|
| 157 |
+
self.last_run_output = result.stdout
|
| 158 |
+
self.last_run_error = result.stderr if result.returncode != 0 else None
|
| 159 |
+
except subprocess.TimeoutExpired:
|
| 160 |
+
self.last_run_error = "TimeoutExpired: script ran longer than 10s."
|
| 161 |
+
self.last_run_output = ""
|
| 162 |
+
finally:
|
| 163 |
+
cleanup()
|
| 164 |
+
|
| 165 |
+
def _grade_current_stage(self) -> float:
|
| 166 |
+
if self.current_stage == 1:
|
| 167 |
+
return grader_stage1.grade(
|
| 168 |
+
self.current_script,
|
| 169 |
+
self.last_run_output,
|
| 170 |
+
self.last_run_error,
|
| 171 |
+
)
|
| 172 |
+
if self.current_stage == 2:
|
| 173 |
+
return grader_stage2.grade(
|
| 174 |
+
self.current_script,
|
| 175 |
+
self.last_run_output,
|
| 176 |
+
self.last_run_error,
|
| 177 |
+
)
|
| 178 |
+
if self.current_stage == 3:
|
| 179 |
+
return grader_stage3.grade(
|
| 180 |
+
self.current_script,
|
| 181 |
+
self._full_df,
|
| 182 |
+
self._held_out_X,
|
| 183 |
+
self._held_out_y,
|
| 184 |
+
)
|
| 185 |
+
if self.current_stage == 4:
|
| 186 |
+
return grader_stage4.grade(
|
| 187 |
+
self.current_script,
|
| 188 |
+
self.dataframe,
|
| 189 |
+
self._deploy_conditions,
|
| 190 |
+
)
|
| 191 |
+
return 0.0
|
| 192 |
+
|
| 193 |
+
def _check_stage_transition(self, score: float) -> str:
|
| 194 |
+
stage = self.current_stage
|
| 195 |
+
|
| 196 |
+
if score >= 0.7:
|
| 197 |
+
if stage not in self.stages_completed:
|
| 198 |
+
self.stages_completed.append(stage)
|
| 199 |
+
self.episode_score = max(0.0, min(1.0, self.episode_score + score * 0.25))
|
| 200 |
+
|
| 201 |
+
if stage == 4:
|
| 202 |
+
self.done = True
|
| 203 |
+
return "Stage 4 complete. Deployment approved."
|
| 204 |
+
|
| 205 |
+
self._load_stage(stage + 1)
|
| 206 |
+
return f"Stage {stage} complete. Advanced to Stage {self.current_stage}."
|
| 207 |
+
|
| 208 |
+
if stage == 4:
|
| 209 |
+
self.loop_count += 1
|
| 210 |
+
self.episode_score = max(0.0, self.episode_score - 0.3)
|
| 211 |
+
if self.loop_count >= 2:
|
| 212 |
+
self.done = True
|
| 213 |
+
return "Stage 4 failed twice. Episode terminated."
|
| 214 |
+
|
| 215 |
+
self._load_stage(1)
|
| 216 |
+
return "Stage 4 failed. Looping back to Stage 1."
|
| 217 |
+
|
| 218 |
+
return f"Stage {stage} score below threshold. Continue fixing this stage."
|
| 219 |
+
|
| 220 |
+
def _handle_query_actor(self) -> None:
|
| 221 |
+
if self.current_stage == 1:
|
| 222 |
+
self.actor_feedback = "No reviewer assigned to this stage."
|
| 223 |
+
elif self.current_stage in {2, 3}:
|
| 224 |
+
self.actor_feedback = self._code_reviewer.review(
|
| 225 |
+
self.current_script,
|
| 226 |
+
stage=self.current_stage,
|
| 227 |
+
)
|
| 228 |
+
elif self.current_stage == 4:
|
| 229 |
+
self.actor_feedback = self._mlops_bot.check_deployment(
|
| 230 |
+
self.current_script,
|
| 231 |
+
self.dataframe,
|
| 232 |
+
self._deploy_conditions,
|
| 233 |
+
)
|
| 234 |
+
|
| 235 |
def step(self, action: Action) -> StepResponse:
|
| 236 |
+
if self.done:
|
| 237 |
+
obs = self._get_observation()
|
| 238 |
+
return StepResponse(
|
| 239 |
+
observation=obs,
|
| 240 |
+
reward=self._make_reward(self.episode_score, "episode already complete", True),
|
| 241 |
+
)
|
| 242 |
+
|
| 243 |
self.step_number += 1
|
| 244 |
+
self.stage_step_number += 1
|
| 245 |
+
|
| 246 |
preview = ""
|
| 247 |
schema = {}
|
| 248 |
score = 0.0
|
| 249 |
+
message = "no reward"
|
| 250 |
+
|
| 251 |
try:
|
| 252 |
if action.action_type == "inspect_data":
|
| 253 |
df = self.dataframe
|
| 254 |
if df is not None:
|
| 255 |
+
preview = (
|
| 256 |
+
f"Shape: {df.shape}\n\n"
|
| 257 |
+
f"Columns: {list(df.columns)}\n\n"
|
| 258 |
+
f"Dtypes:\n{df.dtypes.to_string()}\n\n"
|
| 259 |
+
f"Null counts:\n{df.isnull().sum().to_string()}\n\n"
|
| 260 |
+
f"Sample (5 rows):\n{df.head().to_string()}\n\n"
|
| 261 |
+
f"Numeric stats:\n{df.describe().to_string()}"
|
| 262 |
+
)
|
| 263 |
+
schema = {
|
|
|
|
|
|
|
| 264 |
"columns": list(df.columns),
|
| 265 |
"dtypes": {col: str(df[col].dtype) for col in df.columns},
|
| 266 |
"null_counts": df.isnull().sum().to_dict(),
|
| 267 |
+
"shape": list(df.shape),
|
| 268 |
}
|
| 269 |
+
message = "data inspected"
|
| 270 |
+
|
| 271 |
elif action.action_type == "check_schema":
|
| 272 |
+
if self.dataframe is not None:
|
| 273 |
+
schema = self.dataframe.dtypes.astype(str).to_dict()
|
| 274 |
+
message = "schema checked"
|
| 275 |
+
|
| 276 |
elif action.action_type == "edit_script":
|
| 277 |
old_text = action.payload.get("old", "")
|
| 278 |
new_text = action.payload.get("new", "")
|
| 279 |
+
replacement_script = action.payload.get("script")
|
| 280 |
+
if isinstance(replacement_script, str):
|
| 281 |
+
self.current_script = replacement_script
|
| 282 |
+
self.last_run_error = None
|
| 283 |
+
elif old_text:
|
| 284 |
if old_text not in self.current_script:
|
| 285 |
self.last_run_error = "edit_script failed: the text to replace was not found in the script"
|
| 286 |
else:
|
| 287 |
self.current_script = self.current_script.replace(old_text, new_text)
|
| 288 |
+
self.last_run_error = None
|
| 289 |
+
message = "script edited"
|
| 290 |
+
|
| 291 |
elif action.action_type == "run_script":
|
| 292 |
+
self._run_current_script()
|
| 293 |
+
message = "script ran" if not self.last_run_error else "script failed"
|
| 294 |
+
|
| 295 |
+
elif action.action_type == "query_actor":
|
| 296 |
+
self._handle_query_actor()
|
| 297 |
+
message = "actor queried"
|
| 298 |
+
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 299 |
elif action.action_type == "submit":
|
| 300 |
+
score = self._grade_current_stage()
|
| 301 |
+
message = self._check_stage_transition(score)
|
| 302 |
+
|
| 303 |
+
except Exception as exc:
|
| 304 |
+
self.last_run_error = f"Action Handler Error: {exc}"
|
| 305 |
+
message = self.last_run_error
|
| 306 |
+
|
| 307 |
+
if self.step_number >= 60 or self.stage_step_number >= 15:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 308 |
self.done = True
|
| 309 |
+
message = f"{message} Stage or episode step limit reached."
|
| 310 |
+
|
| 311 |
obs = self._get_observation(partial_schema=schema, partial_preview=preview)
|
| 312 |
+
reward_score = self.episode_score if action.action_type == "submit" else 0.0
|
| 313 |
+
if action.action_type == "submit" and not self.done:
|
| 314 |
+
reward_score = score
|
| 315 |
+
return StepResponse(
|
| 316 |
+
observation=obs,
|
| 317 |
+
reward=self._make_reward(reward_score, message, self.done),
|
| 318 |
+
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 319 |
|
| 320 |
+
def submit_grader(self, task_id: int | None = None) -> Reward:
|
| 321 |
+
score = self._grade_current_stage()
|
| 322 |
+
message = self._check_stage_transition(score)
|
| 323 |
+
return self._make_reward(self.episode_score if self.done else score, message, self.done)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 324 |
|
| 325 |
def state(self) -> StateResponse:
|
| 326 |
return StateResponse(
|
| 327 |
task_id=self.current_task_id,
|
| 328 |
step_number=self.step_number,
|
| 329 |
+
max_steps=60,
|
| 330 |
current_script=self.current_script,
|
| 331 |
episode_done=self.done,
|
| 332 |
+
task_description=self.task_description,
|
| 333 |
+
current_stage=self.current_stage,
|
| 334 |
+
stage_step_number=self.stage_step_number,
|
| 335 |
+
loop_count=self.loop_count,
|
| 336 |
+
stages_completed=list(self.stages_completed),
|
| 337 |
+
episode_score=float(max(0.0, min(1.0, self.episode_score))),
|
| 338 |
+
actor_feedback=self.actor_feedback,
|
| 339 |
)
|
app/main.py
CHANGED
|
@@ -5,7 +5,12 @@ from pydantic import BaseModel
|
|
| 5 |
|
| 6 |
from app.environment import DataEngEnvironment
|
| 7 |
from app.models import Action, Observation, Reward, StepResponse, StateResponse, TaskInfo
|
| 8 |
-
from app.tasks import
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
|
| 10 |
app = FastAPI(title="DataEngEnv API", version="1.0.0")
|
| 11 |
|
|
@@ -35,16 +40,25 @@ class BaselineResponse(BaseModel):
|
|
| 35 |
success: bool
|
| 36 |
output: str
|
| 37 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 38 |
@app.on_event("startup")
|
| 39 |
async def startup_event():
|
| 40 |
print("DataEngEnv ready")
|
| 41 |
-
# Call generate_scenario() on all
|
| 42 |
try:
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
|
|
|
| 46 |
except Exception as e:
|
| 47 |
-
print(f"Warning: A
|
| 48 |
|
| 49 |
|
| 50 |
@app.post("/reset", response_model=Observation | ErrorResponse)
|
|
@@ -72,26 +86,39 @@ async def api_state():
|
|
| 72 |
@app.get("/tasks", response_model=list[TaskInfo] | ErrorResponse)
|
| 73 |
async def api_tasks():
|
| 74 |
try:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 75 |
return [
|
| 76 |
TaskInfo(
|
| 77 |
task_id=1,
|
| 78 |
-
name="
|
| 79 |
difficulty="easy",
|
| 80 |
-
description="
|
| 81 |
action_schema={}
|
| 82 |
),
|
| 83 |
TaskInfo(
|
| 84 |
task_id=2,
|
| 85 |
-
name="
|
| 86 |
difficulty="medium",
|
| 87 |
-
description="
|
| 88 |
action_schema={}
|
| 89 |
),
|
| 90 |
TaskInfo(
|
| 91 |
task_id=3,
|
| 92 |
-
name="
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 93 |
difficulty="hard",
|
| 94 |
-
description="
|
| 95 |
action_schema={}
|
| 96 |
)
|
| 97 |
]
|
|
@@ -103,11 +130,25 @@ async def api_grader(req: TaskIdRequest | None = None):
|
|
| 103 |
try:
|
| 104 |
tid = req.task_id if req is not None and req.task_id is not None else env.current_task_id
|
| 105 |
if tid != env.current_task_id:
|
| 106 |
-
return ErrorResponse(error="task_id mismatch
|
| 107 |
return env.submit_grader(tid)
|
| 108 |
except Exception as e:
|
| 109 |
return ErrorResponse(error=str(e))
|
| 110 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 111 |
@app.get("/baseline", response_model=BaselineResponse | ErrorResponse)
|
| 112 |
async def api_baseline():
|
| 113 |
try:
|
|
|
|
| 5 |
|
| 6 |
from app.environment import DataEngEnvironment
|
| 7 |
from app.models import Action, Observation, Reward, StepResponse, StateResponse, TaskInfo
|
| 8 |
+
from app.tasks import (
|
| 9 |
+
stage1_data_repair,
|
| 10 |
+
stage2_training_monitor,
|
| 11 |
+
stage3_eval_validation,
|
| 12 |
+
stage4_deploy_gate,
|
| 13 |
+
)
|
| 14 |
|
| 15 |
app = FastAPI(title="DataEngEnv API", version="1.0.0")
|
| 16 |
|
|
|
|
| 40 |
success: bool
|
| 41 |
output: str
|
| 42 |
|
| 43 |
+
class PipelineStatusResponse(BaseModel):
|
| 44 |
+
current_stage: int
|
| 45 |
+
loop_count: int
|
| 46 |
+
stages_completed: list[int]
|
| 47 |
+
total_steps: int
|
| 48 |
+
episode_score: float
|
| 49 |
+
actor_feedback: str
|
| 50 |
+
|
| 51 |
@app.on_event("startup")
|
| 52 |
async def startup_event():
|
| 53 |
print("DataEngEnv ready")
|
| 54 |
+
# Call generate_scenario() on all 4 stages to validate they load properly at startup
|
| 55 |
try:
|
| 56 |
+
stage1_data_repair.generate_scenario()
|
| 57 |
+
stage2_training_monitor.generate_scenario()
|
| 58 |
+
stage3_eval_validation.generate_scenario()
|
| 59 |
+
stage4_deploy_gate.generate_scenario()
|
| 60 |
except Exception as e:
|
| 61 |
+
print(f"Warning: A stage failed to load during startup validation: {e}")
|
| 62 |
|
| 63 |
|
| 64 |
@app.post("/reset", response_model=Observation | ErrorResponse)
|
|
|
|
| 86 |
@app.get("/tasks", response_model=list[TaskInfo] | ErrorResponse)
|
| 87 |
async def api_tasks():
|
| 88 |
try:
|
| 89 |
+
scenarios = {
|
| 90 |
+
1: stage1_data_repair.generate_scenario(),
|
| 91 |
+
2: stage2_training_monitor.generate_scenario(),
|
| 92 |
+
3: stage3_eval_validation.generate_scenario(),
|
| 93 |
+
4: stage4_deploy_gate.generate_scenario(),
|
| 94 |
+
}
|
| 95 |
return [
|
| 96 |
TaskInfo(
|
| 97 |
task_id=1,
|
| 98 |
+
name="Stage 1: Data Repair",
|
| 99 |
difficulty="easy",
|
| 100 |
+
description=scenarios[1]["task_description"],
|
| 101 |
action_schema={}
|
| 102 |
),
|
| 103 |
TaskInfo(
|
| 104 |
task_id=2,
|
| 105 |
+
name="Stage 2: Training Monitor",
|
| 106 |
difficulty="medium",
|
| 107 |
+
description=scenarios[2]["task_description"],
|
| 108 |
action_schema={}
|
| 109 |
),
|
| 110 |
TaskInfo(
|
| 111 |
task_id=3,
|
| 112 |
+
name="Stage 3: Eval Validation",
|
| 113 |
+
difficulty="hard",
|
| 114 |
+
description=scenarios[3]["task_description"],
|
| 115 |
+
action_schema={}
|
| 116 |
+
),
|
| 117 |
+
TaskInfo(
|
| 118 |
+
task_id=4,
|
| 119 |
+
name="Stage 4: Deployment Gate",
|
| 120 |
difficulty="hard",
|
| 121 |
+
description=scenarios[4]["task_description"],
|
| 122 |
action_schema={}
|
| 123 |
)
|
| 124 |
]
|
|
|
|
| 130 |
try:
|
| 131 |
tid = req.task_id if req is not None and req.task_id is not None else env.current_task_id
|
| 132 |
if tid != env.current_task_id:
|
| 133 |
+
return ErrorResponse(error="task_id mismatch - call /reset first; the pipeline controls stage advancement")
|
| 134 |
return env.submit_grader(tid)
|
| 135 |
except Exception as e:
|
| 136 |
return ErrorResponse(error=str(e))
|
| 137 |
|
| 138 |
+
@app.get("/pipeline_status", response_model=PipelineStatusResponse | ErrorResponse)
|
| 139 |
+
async def api_pipeline_status():
|
| 140 |
+
try:
|
| 141 |
+
return PipelineStatusResponse(
|
| 142 |
+
current_stage=env.current_stage,
|
| 143 |
+
loop_count=env.loop_count,
|
| 144 |
+
stages_completed=list(env.stages_completed),
|
| 145 |
+
total_steps=env.step_number,
|
| 146 |
+
episode_score=float(env.episode_score),
|
| 147 |
+
actor_feedback=env.actor_feedback,
|
| 148 |
+
)
|
| 149 |
+
except Exception as e:
|
| 150 |
+
return ErrorResponse(error=str(e))
|
| 151 |
+
|
| 152 |
@app.get("/baseline", response_model=BaselineResponse | ErrorResponse)
|
| 153 |
async def api_baseline():
|
| 154 |
try:
|
app/models.py
CHANGED
|
@@ -1,14 +1,13 @@
|
|
| 1 |
from pydantic import BaseModel, Field, field_validator
|
| 2 |
-
import typing
|
| 3 |
|
| 4 |
class Action(BaseModel):
|
| 5 |
-
action_type: str = Field(description="One of: inspect_data | run_script | edit_script | check_schema | submit")
|
| 6 |
payload: dict = Field(description="Action payload details like patch content etc.")
|
| 7 |
|
| 8 |
@field_validator("action_type")
|
| 9 |
@classmethod
|
| 10 |
def validate_action_type(cls, v: str) -> str:
|
| 11 |
-
allowed = {"inspect_data", "run_script", "edit_script", "check_schema", "submit"}
|
| 12 |
if v not in allowed:
|
| 13 |
raise ValueError(f"action_type must be one of {allowed}")
|
| 14 |
return v
|
|
@@ -22,6 +21,11 @@ class Observation(BaseModel):
|
|
| 22 |
data_preview: str = Field(description="Preview of the current Pandas dataframe")
|
| 23 |
schema_info: dict = Field(description="Column types and info")
|
| 24 |
done: bool = Field(description="Whether the environment episode is complete")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 25 |
|
| 26 |
class Reward(BaseModel):
|
| 27 |
score: float = Field(description="Reward score from 0.0 to 1.0")
|
|
@@ -32,8 +36,8 @@ class Reward(BaseModel):
|
|
| 32 |
@field_validator("score")
|
| 33 |
@classmethod
|
| 34 |
def validate_score(cls, v: float) -> float:
|
| 35 |
-
if not (0.0 < v < 1.0):
|
| 36 |
-
raise ValueError("Score must be
|
| 37 |
return v
|
| 38 |
|
| 39 |
class StepResponse(BaseModel):
|
|
@@ -47,6 +51,12 @@ class StateResponse(BaseModel):
|
|
| 47 |
current_script: str = Field(description="Content of the active python script")
|
| 48 |
episode_done: bool = Field(description="Whether the episode is done")
|
| 49 |
task_description: str = Field(description="Task objective description")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 50 |
|
| 51 |
class TaskInfo(BaseModel):
|
| 52 |
task_id: int = Field(description="ID of the task")
|
|
|
|
| 1 |
from pydantic import BaseModel, Field, field_validator
|
|
|
|
| 2 |
|
| 3 |
class Action(BaseModel):
|
| 4 |
+
action_type: str = Field(description="One of: inspect_data | run_script | edit_script | check_schema | query_actor | submit")
|
| 5 |
payload: dict = Field(description="Action payload details like patch content etc.")
|
| 6 |
|
| 7 |
@field_validator("action_type")
|
| 8 |
@classmethod
|
| 9 |
def validate_action_type(cls, v: str) -> str:
|
| 10 |
+
allowed = {"inspect_data", "run_script", "edit_script", "check_schema", "query_actor", "submit"}
|
| 11 |
if v not in allowed:
|
| 12 |
raise ValueError(f"action_type must be one of {allowed}")
|
| 13 |
return v
|
|
|
|
| 21 |
data_preview: str = Field(description="Preview of the current Pandas dataframe")
|
| 22 |
schema_info: dict = Field(description="Column types and info")
|
| 23 |
done: bool = Field(description="Whether the environment episode is complete")
|
| 24 |
+
current_stage: int = Field(default=1, description="Current pipeline stage")
|
| 25 |
+
stage_step_number: int = Field(default=0, description="Step count within current stage")
|
| 26 |
+
actor_feedback: str = Field(default="", description="Most recent actor bot feedback")
|
| 27 |
+
loop_count: int = Field(default=0, description="Number of deployment failure loops")
|
| 28 |
+
stages_completed: list[int] = Field(default_factory=list, description="Completed pipeline stages")
|
| 29 |
|
| 30 |
class Reward(BaseModel):
|
| 31 |
score: float = Field(description="Reward score from 0.0 to 1.0")
|
|
|
|
| 36 |
@field_validator("score")
|
| 37 |
@classmethod
|
| 38 |
def validate_score(cls, v: float) -> float:
|
| 39 |
+
if not (0.0 <= v <= 1.0):
|
| 40 |
+
raise ValueError("Score must be between 0.0 and 1.0 inclusive")
|
| 41 |
return v
|
| 42 |
|
| 43 |
class StepResponse(BaseModel):
|
|
|
|
| 51 |
current_script: str = Field(description="Content of the active python script")
|
| 52 |
episode_done: bool = Field(description="Whether the episode is done")
|
| 53 |
task_description: str = Field(description="Task objective description")
|
| 54 |
+
current_stage: int = Field(default=1, description="Current pipeline stage")
|
| 55 |
+
stage_step_number: int = Field(default=0, description="Step count within current stage")
|
| 56 |
+
loop_count: int = Field(default=0, description="Number of deployment failure loops")
|
| 57 |
+
stages_completed: list[int] = Field(default_factory=list, description="Completed pipeline stages")
|
| 58 |
+
episode_score: float = Field(default=0.0, description="Cumulative 4-stage episode score")
|
| 59 |
+
actor_feedback: str = Field(default="", description="Most recent actor bot feedback")
|
| 60 |
|
| 61 |
class TaskInfo(BaseModel):
|
| 62 |
task_id: int = Field(description="ID of the task")
|
baseline/results.json
CHANGED
|
@@ -1,5 +1,11 @@
|
|
| 1 |
-
|
| 2 |
-
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"final_score": 1.0,
|
| 3 |
+
"total_steps": 18,
|
| 4 |
+
"stages_completed": [
|
| 5 |
+
1,
|
| 6 |
+
2,
|
| 7 |
+
3,
|
| 8 |
+
4
|
| 9 |
+
],
|
| 10 |
+
"episode_score": 1.0
|
| 11 |
+
}
|
baseline/run_baseline.py
CHANGED
|
@@ -91,8 +91,8 @@ def run_episode():
|
|
| 91 |
action = {"action_type": "run_script", "payload": {}}
|
| 92 |
elif current_stage == 3 and stage_step == 1:
|
| 93 |
action = {"action_type": "edit_script", "payload": {
|
| 94 |
-
"old": "scaler = StandardScaler()\nX_scaled = scaler.fit_transform(X)
|
| 95 |
-
"new": "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\nscaler = StandardScaler()\nX_train = scaler.fit_transform(X_train)\nX_test = scaler.transform(X_test)"
|
| 96 |
}}
|
| 97 |
elif current_stage == 3 and stage_step == 2:
|
| 98 |
action = {"action_type": "run_script", "payload": {}}
|
|
|
|
| 91 |
action = {"action_type": "run_script", "payload": {}}
|
| 92 |
elif current_stage == 3 and stage_step == 1:
|
| 93 |
action = {"action_type": "edit_script", "payload": {
|
| 94 |
+
"old": "scaler = StandardScaler()\nX_scaled = scaler.fit_transform(X)\nX_train, X_test, y_train, y_test = train_test_split(\n X_scaled, y, test_size=0.2, random_state=42, stratify=y\n)",
|
| 95 |
+
"new": "X_train, X_test, y_train, y_test = train_test_split(\n X, y, test_size=0.2, random_state=42, stratify=y\n)\nscaler = StandardScaler()\nX_train = scaler.fit_transform(X_train)\nX_test = scaler.transform(X_test)"
|
| 96 |
}}
|
| 97 |
elif current_stage == 3 and stage_step == 2:
|
| 98 |
action = {"action_type": "run_script", "payload": {}}
|
pyproject.toml
CHANGED
|
@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
|
|
| 6 |
name = "dataengenv"
|
| 7 |
version = "1.0.0"
|
| 8 |
description = "ML Data Pipeline Debugger - OpenEnv Environment"
|
| 9 |
-
requires-python = ">=3.10
|
| 10 |
dependencies = [
|
| 11 |
"fastapi==0.115.6",
|
| 12 |
"uvicorn",
|
|
|
|
| 6 |
name = "dataengenv"
|
| 7 |
version = "1.0.0"
|
| 8 |
description = "ML Data Pipeline Debugger - OpenEnv Environment"
|
| 9 |
+
requires-python = ">=3.10"
|
| 10 |
dependencies = [
|
| 11 |
"fastapi==0.115.6",
|
| 12 |
"uvicorn",
|
uv.lock
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|