fix: track missing Codex files (mlops_bot, tasks, graders) and add app/actors/__init__.py to resolve ModuleNotFoundError

app/actors/__init__.py

@@ -0,0 +1 @@
+# Initialize actors package

app/actors/mlops_bot.py

@@ -0,0 +1,142 @@
+from __future__ import annotations
+
+import re
+import shutil
+import subprocess
+import sys
+import uuid
+from pathlib import Path
+
+import pandas as pd
+
+
+FEATURE_COLUMNS = [
+    "age",
+    "salary",
+    "credit_score",
+    "loan_amount",
+    "employment_years",
+]
+
+
+class MLOpsBot:
+    def _parse_value(self, pattern: str, output: str) -> float | None:
+        matches = re.findall(pattern, output, flags=re.IGNORECASE)
+        if not matches:
+            return None
+        try:
+            return float(matches[-1])
+        except ValueError:
+            return None
+
+    def _run_checks(self, script: str, dataframe: pd.DataFrame) -> tuple[float, float]:
+        runs_dir = Path.cwd() / ".actor_runs"
+        runs_dir.mkdir(exist_ok=True)
+        tmpdir = runs_dir / f"mlops_{uuid.uuid4().hex}"
+        tmpdir.mkdir()
+
+        def cleanup() -> None:
+            shutil.rmtree(tmpdir, ignore_errors=True)
+
+        data_path = tmpdir / "data.csv"
+        script_path = tmpdir / "candidate.py"
+
+        try:
+            dataframe.to_csv(data_path, index=False)
+        except Exception:
+            cleanup()
+            return 0.0, 1.0
+
+        postlude = f"""
+from sklearn.metrics import accuracy_score as _accuracy_score
+_deploy_df = pd.read_csv(r'{data_path.as_posix()}')
+if 'X_test' in globals() and 'y_test' in globals():
+    _X_raw = X_test.copy()
+    _y = y_test.copy()
+else:
+    _X_raw = _deploy_df[{FEATURE_COLUMNS!r}]
+    _y = _deploy_df['target']
+_model = globals().get('clf') or globals().get('model') or globals().get('classifier') or globals().get('pipeline')
+if _model is None:
+    raise RuntimeError('No trained model found; expected clf, model, classifier, or pipeline.')
+if 'pipeline' in globals() and _model is globals().get('pipeline'):
+    _pred = _model.predict(_X_raw)
+else:
+    _scaler = globals().get('scaler')
+    _X_eval = _scaler.transform(_X_raw) if _scaler is not None else _X_raw
+    _pred = _model.predict(_X_eval)
+_eval_df = _X_raw.copy()
+_eval_df['target'] = _y
+_eval_df['_pred'] = _pred
+_group_a = _eval_df[_eval_df['age'] < 40]
+_group_b = _eval_df[_eval_df['age'] >= 40]
+_acc_a = _accuracy_score(_group_a['target'], _group_a['_pred'])
+_acc_b = _accuracy_score(_group_b['target'], _group_b['_pred'])
+print('__FAIRNESS_GAP__:', abs(_acc_a - _acc_b))
+"""
+        code = (
+            "import pandas as pd\n"
+            f"df = pd.read_csv(r'{data_path.as_posix()}')\n\n"
+            + (script or "")
+            + "\n"
+            + postlude
+        )
+        script_path.write_text(code, encoding="utf-8")
+
+        try:
+            proc = subprocess.run(
+                [sys.executable, str(script_path)],
+                capture_output=True,
+                text=True,
+                timeout=10,
+                check=False,
+                cwd=tmpdir,
+            )
+        except Exception:
+            cleanup()
+            return 0.0, 1.0
+
+        cleanup()
+
+        if proc.returncode != 0 or (proc.stderr or "").strip():
+            return 0.0, 1.0
+
+        stdout = proc.stdout or ""
+        acc = self._parse_value(r"Accuracy\s*:\s*([0-9]*\.?[0-9]+)", stdout)
+        gap = self._parse_value(r"__FAIRNESS_GAP__\s*:\s*([0-9]*\.?[0-9]+)", stdout)
+        return (acc if acc is not None else 0.0), (gap if gap is not None else 1.0)
+
+    def check_deployment(
+        self,
+        script: str,
+        dataframe: pd.DataFrame,
+        deploy_conditions: dict,
+    ) -> str:
+        threshold = deploy_conditions["accuracy_threshold"]
+        max_ms = deploy_conditions["max_inference_ms"]
+        max_gap = deploy_conditions["max_fairness_gap"]
+        acc, gap = self._run_checks(script, dataframe)
+
+        if acc < threshold:
+            return (
+                "REJECTED: Accuracy below threshold.\n"
+                f" [FAIL] Accuracy: {acc:.2f} < {threshold}\n"
+                " Fix model performance before resubmitting."
+            )
+
+        if gap > max_gap:
+            return (
+                "REJECTED: Fairness constraint violated.\n"
+                f" [PASS] Accuracy: {acc:.2f} >= {threshold}\n"
+                f" [FAIL] Fairness gap: {gap:.3f} > {max_gap}\n"
+                " Model performs unequally across age groups.\n"
+                " Consider class_weight='balanced' or stratified sampling."
+            )
+
+        return (
+            "APPROVED: All deployment conditions met.\n"
+            f" [PASS] Accuracy: {acc:.2f} >= {threshold}\n"
+            f" [PASS] Inference time: 45ms <= {max_ms}ms  \n"
+            f" [PASS] Fairness gap: {gap:.3f} <= {max_gap}\n"
+            " Deploying to production."
+        )

app/graders/grader_stage3.py

@@ -0,0 +1,134 @@
+from __future__ import annotations
+
+import re
+import shutil
+import subprocess
+import sys
+import uuid
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+
+
+FEATURE_COLUMNS = [
+    "age",
+    "salary",
+    "credit_score",
+    "loan_amount",
+    "employment_years",
+]
+
+
+def _detect_fit_after_split(script: str) -> bool:
+    lines = script.split('\n')
+    # Strip import lines and comments entirely
+    code_lines = [
+        l for l in lines 
+        if not l.strip().startswith('import') 
+        and not l.strip().startswith('from')
+        and not l.strip().startswith('#')
+        and l.strip() != ''
+    ]
+    code_only = '\n'.join(code_lines)
+    
+    fit_pos = code_only.find('scaler.fit')
+    split_pos = code_only.find('train_test_split(')
+    
+    # Fix is correct only if train_test_split appears BEFORE scaler.fit in code
+    if fit_pos == -1 or split_pos == -1:
+        return False
+    return split_pos < fit_pos
+
+
+def _parse_marker(output: str) -> float | None:
+    matches = re.findall(r"__HELD_OUT_ACCURACY__\s*:\s*([0-9]*\.?[0-9]+)", output)
+    if not matches:
+        return None
+    try:
+        return float(matches[-1])
+    except ValueError:
+        return None
+
+
+def grade(script: str, full_df: pd.DataFrame, held_out_X, held_out_y) -> float:
+    fit_after_split = _detect_fit_after_split(script or "")
+
+    runs_dir = Path.cwd() / ".grader_runs"
+    runs_dir.mkdir(exist_ok=True)
+    tmpdir = runs_dir / f"stage3_{uuid.uuid4().hex}"
+    tmpdir.mkdir()
+
+    def _cleanup() -> None:
+        shutil.rmtree(tmpdir, ignore_errors=True)
+    data_path = tmpdir / "data.csv"
+    heldout_path = tmpdir / "held_out.csv"
+    script_path = tmpdir / "candidate.py"
+
+    try:
+        full_df.to_csv(data_path, index=False)
+        heldout_df = pd.DataFrame(np.asarray(held_out_X), columns=FEATURE_COLUMNS)
+        heldout_df["target"] = np.asarray(held_out_y)
+        heldout_df.to_csv(heldout_path, index=False)
+    except Exception:
+        _cleanup()
+        return 0.0
+
+    postlude = f"""
+from sklearn.metrics import accuracy_score as _accuracy_score
+_held_df = pd.read_csv(r'{heldout_path.as_posix()}')
+_X_held_raw = _held_df[{FEATURE_COLUMNS!r}]
+_y_held = _held_df['target']
+_model = globals().get('clf') or globals().get('model') or globals().get('classifier') or globals().get('pipeline')
+if _model is None:
+    raise RuntimeError('No trained model found; expected clf, model, classifier, or pipeline.')
+if 'pipeline' in globals() and _model is globals().get('pipeline'):
+    _pred = _model.predict(_X_held_raw)
+else:
+    _scaler = globals().get('scaler')
+    _X_held = _scaler.transform(_X_held_raw) if _scaler is not None else _X_held_raw
+    _pred = _model.predict(_X_held)
+print('__HELD_OUT_ACCURACY__:', _accuracy_score(_y_held, _pred))
+"""
+    code = (
+        "import pandas as pd\n"
+        f"df = pd.read_csv(r'{data_path.as_posix()}')\n\n"
+        + (script or "")
+        + "\n"
+        + postlude
+    )
+    script_path.write_text(code, encoding="utf-8")
+
+    try:
+        proc = subprocess.run(
+            [sys.executable, str(script_path)],
+            capture_output=True,
+            text=True,
+            timeout=10,
+            check=False,
+        )
+    except subprocess.TimeoutExpired:
+        _cleanup()
+        return 0.0
+    except Exception:
+        _cleanup()
+        return 0.0
+
+    _cleanup()
+
+    if proc.returncode != 0 or (proc.stderr or "").strip():
+        return 0.0
+
+    held_out_accuracy = _parse_marker(proc.stdout or "")
+    if held_out_accuracy is None or held_out_accuracy < 0.60:
+        return 0.0
+
+    score = 0.5
+    if fit_after_split:
+        score = max(score, 0.3)
+        if held_out_accuracy >= 0.75:
+            score = 1.0
+        elif held_out_accuracy >= 0.70:
+            score = 0.75
+
+    return float(max(0.0, min(1.0, score)))

app/graders/grader_stage4.py

@@ -0,0 +1,135 @@
+from __future__ import annotations
+
+import re
+import shutil
+import subprocess
+import sys
+import uuid
+from pathlib import Path
+
+import pandas as pd
+
+
+FEATURE_COLUMNS = [
+    "age",
+    "salary",
+    "credit_score",
+    "loan_amount",
+    "employment_years",
+]
+
+
+def _parse_accuracy(output: str) -> float | None:
+    matches = re.findall(r"Accuracy\s*:\s*([0-9]*\.?[0-9]+)", output, flags=re.IGNORECASE)
+    if not matches:
+        return None
+    try:
+        return float(matches[-1])
+    except ValueError:
+        return None
+
+
+def _parse_fairness_gap(output: str) -> float | None:
+    matches = re.findall(r"__FAIRNESS_GAP__\s*:\s*([0-9]*\.?[0-9]+)", output)
+    if not matches:
+        return None
+    try:
+        return float(matches[-1])
+    except ValueError:
+        return None
+
+
+def grade(script: str, dataframe: pd.DataFrame, deploy_conditions: dict) -> float:
+    runs_dir = Path.cwd() / ".grader_runs"
+    runs_dir.mkdir(exist_ok=True)
+    tmpdir = runs_dir / f"stage4_{uuid.uuid4().hex}"
+    tmpdir.mkdir()
+
+    def _cleanup() -> None:
+        shutil.rmtree(tmpdir, ignore_errors=True)
+    data_path = tmpdir / "data.csv"
+    script_path = tmpdir / "candidate.py"
+
+    try:
+        dataframe.to_csv(data_path, index=False)
+    except Exception:
+        _cleanup()
+        return 0.0
+
+    postlude = f"""
+from sklearn.metrics import accuracy_score as _accuracy_score
+_deploy_df = pd.read_csv(r'{data_path.as_posix()}')
+if 'X_test' in globals() and 'y_test' in globals():
+    _X_raw = X_test.copy()
+    _y = y_test.copy()
+else:
+    _X_raw = _deploy_df[{FEATURE_COLUMNS!r}]
+    _y = _deploy_df['target']
+_model = globals().get('clf') or globals().get('model') or globals().get('classifier') or globals().get('pipeline')
+if _model is None:
+    raise RuntimeError('No trained model found; expected clf, model, classifier, or pipeline.')
+if 'pipeline' in globals() and _model is globals().get('pipeline'):
+    _pred = _model.predict(_X_raw)
+else:
+    _scaler = globals().get('scaler')
+    _X_eval = _scaler.transform(_X_raw) if _scaler is not None else _X_raw
+    _pred = _model.predict(_X_eval)
+_eval_df = _X_raw.copy()
+_eval_df['target'] = _y
+_eval_df['_pred'] = _pred
+_group_a = _eval_df[_eval_df['age'] < 40]
+_group_b = _eval_df[_eval_df['age'] >= 40]
+_acc_a = _accuracy_score(_group_a['target'], _group_a['_pred'])
+_acc_b = _accuracy_score(_group_b['target'], _group_b['_pred'])
+print('__FAIRNESS_GAP__:', abs(_acc_a - _acc_b))
+"""
+    code = (
+        "import pandas as pd\n"
+        f"df = pd.read_csv(r'{data_path.as_posix()}')\n\n"
+        + (script or "")
+        + "\n"
+        + postlude
+    )
+    script_path.write_text(code, encoding="utf-8")
+
+    try:
+        proc = subprocess.run(
+            [sys.executable, str(script_path)],
+            capture_output=True,
+            text=True,
+            timeout=10,
+            check=False,
+            cwd=tmpdir,
+        )
+    except subprocess.TimeoutExpired:
+        _cleanup()
+        return 0.0
+    except Exception:
+        _cleanup()
+        return 0.0
+
+    _cleanup()
+
+    if proc.returncode != 0 or (proc.stderr or "").strip():
+        return 0.0
+
+    stdout = proc.stdout or ""
+    accuracy = _parse_accuracy(stdout)
+    fairness_gap = _parse_fairness_gap(stdout)
+    if accuracy is None or fairness_gap is None:
+        return 0.2
+
+    inference_ms = 45
+    accuracy_pass = accuracy >= deploy_conditions["accuracy_threshold"]
+    inference_pass = inference_ms <= deploy_conditions["max_inference_ms"]
+    fairness_pass = fairness_gap <= deploy_conditions["max_fairness_gap"]
+
+    score = 0.2
+    if accuracy_pass:
+        score = 0.4
+    if fairness_pass:
+        score = max(score, 0.7)
+    if accuracy_pass and inference_pass and fairness_pass:
+        score = 1.0
+
+    return float(max(0.0, min(1.0, score)))

app/tasks/stage3_eval_validation.py

@@ -0,0 +1,96 @@
+import numpy as np
+import pandas as pd
+
+
+FEATURE_COLUMNS = [
+    "age",
+    "salary",
+    "credit_score",
+    "loan_amount",
+    "employment_years",
+]
+
+
+def _make_data(rng: np.random.Generator, n: int) -> pd.DataFrame:
+    age = rng.integers(21, 70, size=n).astype(float)
+    salary = np.clip(
+        rng.normal(72_000, 22_000, size=n) + (age - 40) * 600,
+        25_000,
+        170_000,
+    )
+    credit_score = np.clip(
+        rng.normal(675, 70, size=n) + (salary - 72_000) / 2_500 - (age - 40) * 0.6,
+        350,
+        850,
+    )
+    loan_amount = np.clip(
+        rng.normal(24_000, 9_000, size=n) - (credit_score - 675) * 25 + (age - 40) * 120,
+        3_000,
+        65_000,
+    )
+    employment_years = np.clip((age - 22) * 0.45 + rng.normal(0, 4, size=n), 0, 42)
+
+    logits = (
+        (credit_score - 665) * 0.012
+        + (salary - 72_000) / 35_000
+        - (loan_amount - 24_000) / 18_000
+        + employment_years * 0.045
+        - (age - 42) * 0.018
+        + rng.normal(0, 1.15, size=n)
+    )
+    target = (logits > 0.15).astype(int)
+
+    return pd.DataFrame(
+        {
+            "age": age,
+            "salary": salary,
+            "credit_score": credit_score,
+            "loan_amount": loan_amount,
+            "employment_years": employment_years,
+            "target": target,
+        }
+    )
+
+
+def generate_scenario() -> dict:
+    rng = np.random.default_rng(42)
+    df = _make_data(rng, 1000)
+    held_out_df = _make_data(rng, 200)
+
+    held_out_X = held_out_df[FEATURE_COLUMNS].to_numpy()
+    held_out_y = held_out_df["target"].to_numpy()
+
+    broken_script = (
+        "import pandas as pd\n"
+        "from sklearn.ensemble import GradientBoostingClassifier\n"
+        "from sklearn.model_selection import train_test_split\n"
+        "from sklearn.preprocessing import StandardScaler\n"
+        "\n"
+        "X = df[['age', 'salary', 'credit_score', 'loan_amount', 'employment_years']].copy()\n"
+        "y = df['target']\n"
+        "\n"
+        "scaler = StandardScaler()\n"
+        "X_scaled = scaler.fit_transform(X)\n"
+        "X_train, X_test, y_train, y_test = train_test_split(\n"
+        "    X_scaled, y, test_size=0.2, random_state=42, stratify=y\n"
+        ")\n"
+        "\n"
+        "clf = GradientBoostingClassifier(random_state=42)\n"
+        "clf.fit(X_train, y_train)\n"
+        "print('Accuracy: 0.98')\n"
+    )
+
+    return {
+        "broken_script": broken_script,
+        "dataframe": df,
+        "held_out_X": held_out_X,
+        "held_out_y": held_out_y,
+        "full_df": df,
+        "initial_error_log": "",
+        "task_description": (
+            "Model accuracy is 0.98. No errors. But something is wrong. "
+            "Investigate the evaluation pipeline and fix any logical bugs."
+        ),
+        "expected_fix_hint": "move scaler.fit() to after train_test_split()",
+        "stage_number": 3,
+    }

app/tasks/stage4_deploy_gate.py

@@ -0,0 +1,105 @@
+import numpy as np
+import pandas as pd
+
+
+def generate_scenario() -> dict:
+    rng = np.random.default_rng(42)
+    n = 1000
+
+    age = np.concatenate(
+        [
+            rng.integers(22, 40, size=650),
+            rng.integers(40, 70, size=350),
+        ]
+    ).astype(float)
+    rng.shuffle(age)
+    older_group = (age >= 40).astype(float)
+
+    salary = np.clip(
+        rng.normal(68_000, 18_000, size=n) + (age - 40) * 900,
+        25_000,
+        180_000,
+    )
+    credit_score = np.clip(
+        rng.normal(665, 65, size=n) + (salary - 68_000) / 3_000 - older_group * 40,
+        350,
+        850,
+    )
+    loan_amount = np.clip(
+        rng.normal(23_000, 9_000, size=n) - (credit_score - 665) * 20 + older_group * 3_000,
+        3_000,
+        70_000,
+    )
+    employment_years = np.clip((age - 22) * 0.45 + rng.normal(0, 4, size=n), 0, 45)
+
+    logits = (
+        (credit_score - 660) * 0.012
+        + (salary - 68_000) / 35_000
+        - (loan_amount - 23_000) / 18_000
+        + employment_years * 0.035
+        - older_group * 0.7
+        + rng.normal(0, 0.9, size=n)
+    )
+    target = (logits > 0.3).astype(int)
+
+    df = pd.DataFrame(
+        {
+            "age": age,
+            "salary": salary,
+            "credit_score": credit_score,
+            "loan_amount": loan_amount,
+            "employment_years": employment_years,
+            "target": target,
+        }
+    )
+
+    broken_script = (
+        "import pandas as pd\n"
+        "from sklearn.linear_model import LogisticRegression\n"
+        "from sklearn.metrics import accuracy_score\n"
+        "from sklearn.model_selection import train_test_split\n"
+        "from sklearn.preprocessing import StandardScaler\n"
+        "\n"
+        "features = ['age', 'salary', 'credit_score', 'loan_amount', 'employment_years']\n"
+        "X = df[features].copy()\n"
+        "y = df['target']\n"
+        "X_train, X_test, y_train, y_test = train_test_split(\n"
+        "    X, y, test_size=0.25, random_state=42, stratify=y\n"
+        ")\n"
+        "\n"
+        "scaler = StandardScaler()\n"
+        "X_train_scaled = scaler.fit_transform(X_train)\n"
+        "X_test_scaled = scaler.transform(X_test)\n"
+        "\n"
+        "clf = LogisticRegression(max_iter=1000, random_state=42)\n"
+        "clf.fit(X_train_scaled, y_train)\n"
+        "pred = clf.predict(X_test_scaled)\n"
+        "print('Accuracy:', round(accuracy_score(y_test, pred), 2))\n"
+    )
+
+    return {
+        "broken_script": broken_script,
+        "dataframe": df,
+        "deploy_conditions": {
+            "accuracy_threshold": 0.75,
+            "max_inference_ms": 100,
+            "max_fairness_gap": 0.05,
+        },
+        "initial_error_log": (
+            "DEPLOY GATE CHECK:\n"
+            "[PASS] Accuracy: 0.77 >= 0.75\n"
+            "[PASS] Inference time: 45ms <= 100ms\n"
+            "[FAIL] Fairness gap: 0.09 > 0.05 (age groups differ significantly)\n"
+            "STATUS: REJECTED \u2014 fix fairness issue before deployment"
+        ),
+        "task_description": (
+            "Deployment rejected. Fix the fairness issue. "
+            "The model performs significantly worse on one age group. "
+            "All 3 conditions must pass for deployment approval."
+        ),
+        "expected_fix_hint": (
+            "add class_weight='balanced' to classifier OR "
+            "add age-group stratified sampling in train_test_split"
+        ),
+        "stage_number": 4,
+    }