CoBeDigger commited on
Commit
051d18c
·
1 Parent(s): 61b8743

feat(round2): build Stage 1 Data Repair task and string-based deterministic grader

Browse files
app/graders/grader_stage1.py ADDED
@@ -0,0 +1,69 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import re
4
+ from typing import Optional
5
+
6
+
7
+ def _contains_accuracy(output: str) -> bool:
8
+ """Check if output contains a printed accuracy value."""
9
+ return bool(re.search(r"Accuracy\s*:\s*\d*\.?\d+", output, flags=re.IGNORECASE))
10
+
11
+
12
+ def _has_nan_handling(script: str) -> bool:
13
+ """Detect common NaN/missing-value handling patterns in script."""
14
+ s = script.lower()
15
+ return any(
16
+ k in s for k in ["dropna", "fillna", "simpleimputer", "isna()", "notna()"]
17
+ )
18
+
19
+
20
+ def _has_outlier_handling(script: str) -> bool:
21
+ """Detect common outlier handling patterns in script."""
22
+ s = script.lower()
23
+ return any(
24
+ k in s for k in ["clip(", "quantile(", "iqr", "winsor", "robustscaler"]
25
+ )
26
+
27
+
28
+ def grade(script: str, run_output: str, run_error: Optional[str]) -> float:
29
+ """
30
+ Stage 1 Grader — Data Repair.
31
+
32
+ Scoring ladder (deterministic, string-based only, no script execution):
33
+ 0.0 → KeyError still present (age_years in script)
34
+ 0.2 → age_years fixed but NaN error still occurs
35
+ 0.4 → script contains NaN handling (dropna/fillna/etc.)
36
+ 0.6 → script contains outlier handling (clip/quantile/etc.)
37
+ 0.8 → script runs without any error
38
+ 1.0 → script runs clean AND prints accuracy value
39
+
40
+ Checks from top (1.0) downward, returns first match.
41
+ """
42
+ run_error = run_error or ""
43
+ run_output = run_output or ""
44
+ script = script or ""
45
+
46
+ # ── Hard failure: column rename bug still present ──
47
+ if "age_years" in script:
48
+ return 0.0
49
+
50
+ # ── Top tier: clean run with accuracy printed ──
51
+ if run_error.strip() == "" and _contains_accuracy(run_output):
52
+ return 1.0
53
+
54
+ # ── Clean run but no accuracy printed ──
55
+ if run_error.strip() == "":
56
+ return 0.8
57
+
58
+ # ── Script still has errors, but check for fix progress ──
59
+
60
+ # Outlier handling present (implies NaN handling also likely attempted)
61
+ if _has_outlier_handling(script) and _has_nan_handling(script):
62
+ return 0.6
63
+
64
+ # NaN handling present
65
+ if _has_nan_handling(script):
66
+ return 0.4
67
+
68
+ # Column rename fixed but script still crashes (NaN error)
69
+ return 0.2
app/tasks/stage1_data_repair.py ADDED
@@ -0,0 +1,98 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import numpy as np
2
+ import pandas as pd
3
+
4
+
5
+ def generate_scenario() -> dict:
6
+ """
7
+ Stage 1 — Data Repair (combines old Task 1 + Task 2).
8
+
9
+ Two-bug cascade:
10
+ Bug 1: Script references 'age_years' but DataFrame column is 'age' → KeyError
11
+ Bug 2: NaN values in data cause StandardScaler to crash → ValueError
12
+
13
+ The agent must make TWO edit_script actions to fix both bugs sequentially.
14
+
15
+ Data: 500 rows, seed=42, with ~15% NaN injection and 10 salary outliers.
16
+ """
17
+
18
+ rng = np.random.default_rng(42)
19
+ n = 500
20
+
21
+ age = rng.integers(18, 71, size=n).astype(float)
22
+ salary = np.clip(rng.normal(70000, 20000, size=n), 20000, None)
23
+ credit_score = np.clip(rng.normal(680, 80, size=n), 300, 850)
24
+ loan_amount = np.clip(rng.normal(20000, 10000, size=n), 1000, None)
25
+ employment_years = np.clip(rng.normal(8, 5, size=n), 0, 40).astype(float)
26
+
27
+ # Binary target influenced by features with noise
28
+ logits = (
29
+ (age - 40) * -0.03
30
+ + (salary - 70000) / 50000
31
+ + (credit_score - 650) * 0.004
32
+ - (loan_amount - 20000) / 40000
33
+ + (employment_years - 8) * 0.05
34
+ + rng.normal(0, 0.8, size=n)
35
+ )
36
+ prob = 1 / (1 + np.exp(-logits))
37
+ target = (prob > 0.5).astype(int)
38
+
39
+ df = pd.DataFrame(
40
+ {
41
+ "age": age,
42
+ "salary": salary,
43
+ "credit_score": credit_score,
44
+ "loan_amount": loan_amount,
45
+ "employment_years": employment_years,
46
+ "target": target,
47
+ }
48
+ )
49
+
50
+ # Inject ~15% rows with NaNs across a subset of columns
51
+ nan_rows = rng.choice(np.arange(n), size=int(0.15 * n), replace=False)
52
+ for col in ["age", "credit_score", "employment_years"]:
53
+ df.loc[nan_rows, col] = np.nan
54
+
55
+ # Inject ~10 extreme salary outliers (non-overlapping with NaN rows)
56
+ outlier_rows = rng.choice(
57
+ np.setdiff1d(np.arange(n), nan_rows), size=10, replace=False
58
+ )
59
+ df.loc[outlier_rows, "salary"] = 9_999_999.0
60
+
61
+ broken_script = (
62
+ "import pandas as pd\n"
63
+ "from sklearn.preprocessing import StandardScaler\n"
64
+ "from sklearn.linear_model import LogisticRegression\n"
65
+ "from sklearn.model_selection import train_test_split\n"
66
+ "\n"
67
+ "X = df[['age_years', 'salary', 'credit_score', 'loan_amount', 'employment_years']].copy()\n"
68
+ "y = df['target']\n"
69
+ "scaler = StandardScaler()\n"
70
+ "X_scaled = scaler.fit_transform(X)\n"
71
+ "X_train, X_test, y_train, y_test = train_test_split(X_scaled, y, test_size=0.2, random_state=42)\n"
72
+ "from sklearn.linear_model import LogisticRegression\n"
73
+ "clf = LogisticRegression(max_iter=1000, random_state=42)\n"
74
+ "clf.fit(X_train, y_train)\n"
75
+ "print('Accuracy:', clf.score(X_test, y_test))\n"
76
+ )
77
+
78
+ initial_error_log = (
79
+ "Traceback (most recent call last):\n"
80
+ " File '<script>', line 6, in <module>\n"
81
+ "KeyError: 'age_years'\n"
82
+ )
83
+
84
+ task_description = (
85
+ "Fix the broken pipeline: there are multiple bugs. "
86
+ "Find and fix all of them."
87
+ )
88
+
89
+ expected_fix_hint = "fix age_years→age AND add dropna/clip before scaler"
90
+
91
+ return {
92
+ "broken_script": broken_script,
93
+ "dataframe": df,
94
+ "initial_error_log": initial_error_log,
95
+ "task_description": task_description,
96
+ "expected_fix_hint": expected_fix_hint,
97
+ "stage_number": 1,
98
+ }