Spaces:
Sleeping
Sleeping
File size: 3,574 Bytes
4189f0e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 | import numpy as np
import pandas as pd
def _make_data(rng: np.random.Generator, n: int) -> tuple[pd.DataFrame, np.ndarray]:
"""Generate a dataset with moderate separability so that proper training yields ~0.72-0.78 accuracy.
Deterministic given rng.
Returns (df, y).
"""
# Binary labels 0/1
y = rng.integers(0, 2, size=n)
# Features: class-dependent means with substantial noise
# This yields a problem that is learnable but not near-perfect.
shift = (y * 2 - 1).reshape(-1, 1) # -1 for class 0, +1 for class 1
base = rng.normal(0, 1.5, size=(n, 5))
means = np.array([0.8, 0.6, 0.5, 0.3, 0.0])
X = base + shift * means # class-separable signal
# Add one weakly informative interaction-like feature
x_extra = (X[:, 0] * 0.3 + X[:, 1] * -0.2 + rng.normal(0, 0.7, size=n)).reshape(-1, 1)
X = np.concatenate([X, x_extra], axis=1)
df = pd.DataFrame(X, columns=[
"feat_1",
"feat_2",
"feat_3",
"feat_4",
"feat_5",
"feat_6",
])
df["target"] = y
return df, y
def generate_scenario() -> dict:
"""
Task 3 (Hard): Data leakage via scaling before split.
- Data: ~1000 rows; model accuracy appears suspiciously high when StandardScaler is fit on the full dataset
- Bug: StandardScaler.fit_transform applied before train_test_split (data leakage)
- True accuracy after fixing scaling (fit on train only) should drop to ~0.72-0.78
- Also returns held_out_X, held_out_y (NOT to be exposed externally)
- Deterministic via seed=42
"""
rng = np.random.default_rng(42)
# Main dataset for the episode
df, y = _make_data(rng, 1000)
# Held-out set for grader3 only
held_out_df, held_out_y = _make_data(rng, 200)
held_out_X = held_out_df.drop(columns=["target"]).to_numpy()
broken_script = (
"# Broken: fits StandardScaler on FULL dataset before splitting (data leakage)\n"
"import numpy as np\n"
"from sklearn.preprocessing import StandardScaler\n"
"from sklearn.linear_model import LogisticRegression\n"
"from sklearn.model_selection import train_test_split\n"
"\n"
"X = df.drop(columns=['target']).to_numpy()\n"
"y = df['target'].to_numpy()\n"
"scaler = StandardScaler()\n"
"X_scaled = scaler.fit_transform(X) # data leakage: fit on full data\n"
"X_train, X_test, y_train, y_test = train_test_split(X_scaled, y, test_size=0.2, random_state=42)\n"
"clf = LogisticRegression(max_iter=1000, random_state=42)\n"
"clf.fit(X_train, y_train)\n"
"acc = clf.score(X_test, y_test)\n"
"print('Accuracy:', round(acc, 4))\n"
)
# No error on run; it will print an accuracy that looks inflated.
initial_error_log = ""
task_description = (
"The script performs StandardScaler.fit_transform on the entire dataset before splitting into "
"train/test, leaking information. This yields a suspiciously high reported accuracy."
)
expected_fix_hint = (
"Move scaler.fit to occur AFTER train_test_split and fit ONLY on X_train, then transform X_train and X_test."
)
return {
"broken_script": broken_script,
"dataframe": df,
"initial_error_log": initial_error_log,
"task_description": task_description,
"expected_fix_hint": expected_fix_hint,
# Hidden extras for grader3 (never exposed via API)
"held_out_X": held_out_X,
"held_out_y": held_out_y,
"full_df": df,
}
|